From 2d753c390688aebbbdd5e455e8abdc898aa98a7a Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 28 Oct 2026 21:13:15 -0700 Subject: [PATCH 01/12] fix(agentic): surface a failed agentic follow-up call instead of the pre-loop tool call --- .../chat_completion_agentic_loop.py | 72 +++--- litellm/llms/custom_httpx/llm_http_handler.py | 218 ++++++++---------- .../test_websearch_agentic_loop_cap.py | 44 ++++ .../test_websearch_thinking_constraint.py | 3 +- .../test_chat_completion_agentic_loop.py | 97 ++++++++ .../custom_httpx/test_llm_http_handler.py | 114 +++++++++ 6 files changed, 391 insertions(+), 157 deletions(-) diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 628486f377f..a43bb6c07b9 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -304,21 +304,8 @@ async def maybe_run_chat_completion_agentic_loop( model=model, ) - try: - if not _build_plan_overridden(callback): - return await callback.async_run_agentic_loop( - tools=tool_calls, - model=model, - messages=messages, - response=response, - anthropic_messages_provider_config=None, - anthropic_messages_optional_request_params=optional_params, - logging_obj=logging_obj, - stream=stream, - kwargs=hook_kwargs, - ) - - plan = await callback.async_build_agentic_loop_plan( + if not _build_plan_overridden(callback): + return await callback.async_run_agentic_loop( tools=tool_calls, model=model, messages=messages, @@ -330,31 +317,38 @@ async def maybe_run_chat_completion_agentic_loop( kwargs=hook_kwargs, ) - if plan.response_override is not None: - return plan.response_override - if plan.terminate: - return response - if not plan.run_agentic_loop: - continue + plan = await callback.async_build_agentic_loop_plan( + tools=tool_calls, + model=model, + messages=messages, + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs=hook_kwargs, + ) - return await _execute_chat_completion_agentic_plan( - plan=plan, - callback=callback, - model=model, - optional_params=optional_params, - kwargs=kwargs, - logging_obj=logging_obj, - custom_llm_provider=custom_llm_provider, - depth=depth, - max_loops=max_loops, - fingerprints=fingerprints, - fingerprint=fingerprint, - ) - except Exception as e: - verbose_logger.exception( - "LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s", - str(e), - ) + if plan.response_override is not None: + return plan.response_override + if plan.terminate: + return response + if not plan.run_agentic_loop: + continue + + return await _execute_chat_completion_agentic_plan( + plan=plan, + callback=callback, + model=model, + optional_params=optional_params, + kwargs=kwargs, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + ) return _converted_stream_replay( response, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index e7da6586b97..d659bd75829 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6353,27 +6353,12 @@ class BaseLLMHTTPHandler: api_surface, ) - try: - kwargs_with_provider = hook_kwargs.copy() - kwargs_with_provider["custom_llm_provider"] = custom_llm_provider - build_plan_overridden = ( - callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan - ) - if not build_plan_overridden: - agentic_result: object = await callback.async_run_agentic_loop( - tools=tool_calls, - model=model, - messages=messages, - response=response, - anthropic_messages_provider_config=anthropic_messages_provider_config, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, - logging_obj=logging_obj, - stream=stream, - kwargs=kwargs_with_provider, - ) - return self._maybe_wrap_in_fake_stream(agentic_result, logging_obj, api_surface) - - plan = await callback.async_build_agentic_loop_plan( + kwargs_with_provider = {**hook_kwargs, "custom_llm_provider": custom_llm_provider} + build_plan_overridden = ( + callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan + ) + if not build_plan_overridden: + agentic_result: object = await callback.async_run_agentic_loop( tools=tool_calls, model=model, messages=messages, @@ -6384,68 +6369,73 @@ class BaseLLMHTTPHandler: stream=stream, kwargs=kwargs_with_provider, ) + return self._maybe_wrap_in_fake_stream(agentic_result, logging_obj, api_surface) - if plan.response_override is not None: - return self._maybe_wrap_in_fake_stream(plan.response_override, logging_obj, api_surface) - if plan.terminate: - verbose_logger.debug( - "Agentic loop terminated by callback=%s reason=%s", - callback.__class__.__name__, - plan.stop_reason, - ) - return self._maybe_wrap_in_fake_stream( - await callback.async_post_agentic_loop_response_hook( - response=self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls), - plan=plan, - kwargs=kwargs_with_provider, - ), - logging_obj, - api_surface, - ) - if not plan.run_agentic_loop: - continue - - if api_surface == "responses": - return await self._execute_responses_agentic_plan( - plan=plan, - model=model, - response_api_optional_request_params=anthropic_messages_optional_request_params, - logging_obj=logging_obj, - kwargs=kwargs_with_provider, - depth=depth, - max_loops=max_loops, - fingerprints=fingerprints, - fingerprint=fingerprint, - callback=callback, - ) + plan = await callback.async_build_agentic_loop_plan( + tools=tool_calls, + model=model, + messages=messages, + response=response, + anthropic_messages_provider_config=anthropic_messages_provider_config, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + logging_obj=logging_obj, + stream=stream, + kwargs=kwargs_with_provider, + ) + if plan.response_override is not None: + return self._maybe_wrap_in_fake_stream(plan.response_override, logging_obj, api_surface) + if plan.terminate: + verbose_logger.debug( + "Agentic loop terminated by callback=%s reason=%s", + callback.__class__.__name__, + plan.stop_reason, + ) return self._maybe_wrap_in_fake_stream( - await self._execute_anthropic_agentic_plan( + await callback.async_post_agentic_loop_response_hook( + response=self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls), plan=plan, - model=model, - messages=messages, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, - logging_obj=logging_obj, kwargs=kwargs_with_provider, - depth=depth, - max_loops=max_loops, - fingerprints=fingerprints, - fingerprint=fingerprint, - stream=stream, - callback=callback, ), logging_obj, api_surface, ) - except Exception as e: - _call_id = getattr(logging_obj, "litellm_call_id", "unknown") - verbose_logger.exception( - "LiteLLM.AgenticHookError: Exception in agentic completion hooks [call_id=%s model=%s]: %s", - _call_id, - model, - str(e), + if not plan.run_agentic_loop: + continue + + if api_surface == "responses": + return await self._execute_responses_agentic_plan( + plan=plan, + model=model, + response_api_optional_request_params=anthropic_messages_optional_request_params, + logging_obj=logging_obj, + kwargs=kwargs_with_provider, + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + callback=callback, ) + return self._maybe_wrap_in_fake_stream( + await self._execute_anthropic_agentic_plan( + plan=plan, + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + logging_obj=logging_obj, + kwargs=kwargs_with_provider, + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + stream=stream, + callback=callback, + ), + logging_obj, + api_surface, + ) + # Check if we need to convert response to fake stream # This happens when: # 1. Stream was originally True but converted to False for WebSearch interception @@ -6524,26 +6514,13 @@ class BaseLLMHTTPHandler: model=model, ) - try: - kwargs_with_provider = kwargs.copy() if kwargs else {} - kwargs_with_provider["custom_llm_provider"] = custom_llm_provider - build_plan_overridden = ( - callback.__class__.async_build_chat_completion_agentic_loop_plan - is not CustomLogger.async_build_chat_completion_agentic_loop_plan - ) - if not build_plan_overridden: - return await callback.async_run_chat_completion_agentic_loop( - tools=tool_calls, - model=model, - messages=messages, - response=response, - optional_params=optional_params, - logging_obj=logging_obj, - stream=stream, - kwargs=kwargs_with_provider, - ) - - plan = await callback.async_build_chat_completion_agentic_loop_plan( + kwargs_with_provider = {**(kwargs or {}), "custom_llm_provider": custom_llm_provider} + build_plan_overridden = ( + callback.__class__.async_build_chat_completion_agentic_loop_plan + is not CustomLogger.async_build_chat_completion_agentic_loop_plan + ) + if not build_plan_overridden: + return await callback.async_run_chat_completion_agentic_loop( tools=tool_calls, model=model, messages=messages, @@ -6554,32 +6531,41 @@ class BaseLLMHTTPHandler: kwargs=kwargs_with_provider, ) - if plan.response_override is not None: - return plan.response_override - if plan.terminate: - verbose_logger.debug( - "Agentic chat loop terminated by callback=%s reason=%s", - callback.__class__.__name__, - plan.stop_reason, - ) - return response - if not plan.run_agentic_loop: - continue + plan = await callback.async_build_chat_completion_agentic_loop_plan( + tools=tool_calls, + model=model, + messages=messages, + response=response, + optional_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs=kwargs_with_provider, + ) - return await self._execute_chat_completion_agentic_plan( - plan=plan, - model=model, - messages=messages, - optional_params=optional_params, - kwargs=kwargs_with_provider, - custom_llm_provider=custom_llm_provider, - depth=depth, - max_loops=max_loops, - fingerprints=fingerprints, - fingerprint=fingerprint, + if plan.response_override is not None: + return plan.response_override + if plan.terminate: + verbose_logger.debug( + "Agentic chat loop terminated by callback=%s reason=%s", + callback.__class__.__name__, + plan.stop_reason, ) - except Exception as e: - verbose_logger.exception("LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: %s", e) + return response + if not plan.run_agentic_loop: + continue + + return await self._execute_chat_completion_agentic_plan( + plan=plan, + model=model, + messages=messages, + optional_params=optional_params, + kwargs=kwargs_with_provider, + custom_llm_provider=custom_llm_provider, + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + ) # Check if we need to convert response to fake stream for chat completions # This happens when: diff --git a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py index 351ca9beb4d..4449c7e7013 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py @@ -509,6 +509,50 @@ def _response_asking_for_searches(*queries: str) -> dict: } +class TestFailedFollowUpCallReachesTheClient: + """ + A follow-up model call that fails after a successful search used to be + logged and dropped, so the client got the pre-loop turn holding a + litellm_web_search tool_use it never declared. + """ + + def setup_method(self): + self.handler = BaseLLMHTTPHandler() + self.logger = WebSearchInterceptionLogger(enabled_providers=["anthropic"]) + + @staticmethod + async def _rate_limited_follow_up(**call_kwargs): + raise RateLimitError( + "UserByModelByMinuteUncachedInputTokens", llm_provider="anthropic", model="claude-sonnet-4-5" + ) + + @staticmethod + async def _search(query, kwargs=None, rich=None): + found = SearchResult(title="Result", url="https://example.com", snippet="A result.", date=None) + return ("Title: Result\nURL: https://example.com", SearchResponse(results=[found])) + + @pytest.mark.asyncio + @pytest.mark.parametrize("converted_stream", (False, True), ids=("non_streaming", "streaming_client")) + async def test_follow_up_rate_limit_is_raised_to_the_caller(self, monkeypatch, converted_stream): + monkeypatch.setattr("litellm.anthropic_interface.messages.acreate", self._rate_limited_follow_up) + + with ( + patch.object(self.logger, "_execute_search", side_effect=self._search), + pytest.raises(RateLimitError, match="UserByModelByMinuteUncachedInputTokens"), + ): + await self.handler._call_agentic_completion_hooks( + response=_response_asking_for_searches("who won the world cup"), + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={"tools": [get_litellm_web_search_tool()]}, + logging_obj=_logging_obj(self.logger, converted_stream=converted_stream), + stream=False, + custom_llm_provider="anthropic", + kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True}, + ) + + class TestFailedSearchEndsTheTurn: """ A search that failed used to come back to the client as an empty successful diff --git a/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py b/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py index b2d5225070c..ee4c4d32478 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_thinking_constraint.py @@ -371,8 +371,7 @@ class TestLoggingObjExcludedFromFollowUp: class TestFollowUpErrorScenarios: """Regression tests: the agentic loop must surface errors properly and - not silently swallow them (except at the _call_agentic_completion_hooks - level which intentionally catches to return the initial response).""" + not silently swallow them.""" @pytest.mark.asyncio async def test_followup_400_raises(self): diff --git a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py index 3cbc3591009..4551c0d22fb 100644 --- a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -466,6 +466,103 @@ async def test_dispatcher_raises_on_repeated_tool_call_fingerprint(restore_callb acompletion_mock.assert_not_awaited() +@pytest.mark.asyncio +async def test_dispatcher_surfaces_a_failed_follow_up_instead_of_the_tool_call_turn(): + logger: Final = _GateOnlyLogger( + plan=AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=_patched_messages(), kwargs={"mock_response": "litellm.RateLimitError"} + ), + ), + tool_calls={"tool_calls": [{"id": "call_abc"}]}, + ) + + with pytest.raises(litellm.RateLimitError): + await _dispatch_tool_call_turn(_tool_call_model_response(), logger) + + assert logger.cleanup_calls == 1 + + +class _RunOnlyLoopLogger(CustomLogger): + def __init__(self, result: ModelResponse) -> None: + super().__init__() + self._result = result + + async def async_should_run_agentic_loop( + self, + response: Any, + model: str, + messages: List[Dict[str, Any]], + tools: Optional[List[Dict[str, Any]]], + stream: bool, + custom_llm_provider: str, + kwargs: Dict[str, Any], + ) -> Tuple[bool, Dict[str, Any]]: + return True, {"tool_calls": [{"id": "call_abc"}]} + + async def async_run_agentic_loop( + self, + tools: Dict[str, Any], + model: str, + messages: List[Dict[str, Any]], + response: Any, + anthropic_messages_provider_config: Any, + anthropic_messages_optional_request_params: Dict[str, Any], + logging_obj: Any, + stream: bool, + kwargs: Dict[str, Any], + ) -> Any: + return self._result + + +class _LoggingWithCallbacks(_LoggingStub): + def __init__(self, callbacks: Tuple[CustomLogger, ...]) -> None: + self.dynamic_success_callbacks = list(callbacks) + + +async def _dispatch_tool_call_turn( + response: ModelResponse, *callbacks: CustomLogger +) -> ModelResponse | CustomStreamWrapper | None: + return await maybe_run_chat_completion_agentic_loop( + response=response, + model="gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + optional_params={}, + kwargs={}, + logging_obj=_LoggingWithCallbacks(callbacks), + custom_llm_provider="openai", + stream=False, + ) + + +@pytest.mark.asyncio +async def test_dispatcher_returns_the_tool_call_turn_when_the_plan_terminates(): + response: Final = _tool_call_model_response() + terminating: Final = _GateOnlyLogger( + plan=AgenticLoopPlan(terminate=True), tool_calls={"tool_calls": [{"id": "call_abc"}]} + ) + + assert await _dispatch_tool_call_turn(response, terminating) is response + + +@pytest.mark.asyncio +async def test_dispatcher_moves_past_a_plan_that_declines_the_loop(): + override: Final = _plain_model_response("from the next callback") + tool_calls: Final = {"tool_calls": [{"id": "call_abc"}]} + declined: Final = _GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=False), tool_calls=tool_calls) + overriding: Final = _GateOnlyLogger(plan=AgenticLoopPlan(response_override=override), tool_calls=tool_calls) + + assert await _dispatch_tool_call_turn(_tool_call_model_response(), declined, overriding) is override + + +@pytest.mark.asyncio +async def test_dispatcher_returns_a_run_only_loops_result(): + loop_result: Final = _plain_model_response() + + assert await _dispatch_tool_call_turn(_tool_call_model_response(), _RunOnlyLoopLogger(loop_result)) is loop_result + + @pytest.mark.asyncio @pytest.mark.parametrize("include_usage", (True, False), ids=("usage_requested", "usage_not_requested")) @pytest.mark.parametrize( diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index f10f5a67b2f..ec52c7b5bc2 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -64,6 +64,7 @@ from litellm.responses.streaming_iterator import ( BaseResponsesAPIStreamingIterator, MockResponsesAPIStreamingIterator, ) +from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( @@ -4992,6 +4993,119 @@ async def test_responses_agentic_followup_sends_the_plans_request_param_over_a_s assert followup_calls[0]["prompt_cache_key"] == "from-plan-params" +class _PlannedChatLoop(CustomLogger): + def __init__(self, plan: AgenticLoopPlan) -> None: + super().__init__() + self._plan = plan + + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: list[dict], + tools: list[dict] | None, + stream: bool, + custom_llm_provider: str, + kwargs: dict, + ) -> tuple[bool, dict]: + return True, {"tool_calls": [{"id": "call_search", "name": "litellm_web_search"}]} + + async def async_build_chat_completion_agentic_loop_plan( + self, + tools: dict, + model: str, + messages: list[dict], + response: object, + optional_params: dict, + logging_obj: object, + stream: bool, + kwargs: dict, + ) -> AgenticLoopPlan: + return self._plan + + +class _RunOnlyChatLoop(CustomLogger): + def __init__(self, result: ModelResponse) -> None: + super().__init__() + self._result = result + + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: list[dict], + tools: list[dict] | None, + stream: bool, + custom_llm_provider: str, + kwargs: dict, + ) -> tuple[bool, dict]: + return True, {"tool_calls": [{"id": "call_search", "name": "litellm_web_search"}]} + + async def async_run_chat_completion_agentic_loop( + self, + tools: dict, + model: str, + messages: list[dict], + response: object, + optional_params: dict, + logging_obj: object, + stream: bool, + kwargs: dict, + ) -> object: + return self._result + + +async def _run_chat_completion_agentic_hooks(response: ModelResponse, *callbacks: CustomLogger) -> object: + logging_obj: Final = Mock(litellm_call_id="call-1", dynamic_success_callbacks=list(callbacks), model_call_details={}) + return await BaseLLMHTTPHandler()._call_agentic_chat_completion_hooks( + response=response, + model="gpt-4o-mini", + messages=[{"role": "user", "content": "what is new in litellm?"}], + optional_params={}, + logging_obj=logging_obj, + stream=False, + custom_llm_provider="openai", + kwargs={}, + ) + + +@pytest.mark.asyncio +async def test_chat_completion_agentic_hooks_surface_a_failed_follow_up_instead_of_the_tool_call_turn(): + rate_limited_follow_up: Final = AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[{"role": "user", "content": "here are the search results"}], + kwargs={"mock_response": "litellm.RateLimitError"}, + ), + ) + + with pytest.raises(litellm.RateLimitError): + await _run_chat_completion_agentic_hooks(ModelResponse(), _PlannedChatLoop(rate_limited_follow_up)) + + +@pytest.mark.asyncio +async def test_chat_completion_agentic_hooks_return_the_tool_call_turn_when_the_plan_terminates(): + response: Final = ModelResponse() + + assert await _run_chat_completion_agentic_hooks(response, _PlannedChatLoop(AgenticLoopPlan(terminate=True))) is response + + +@pytest.mark.asyncio +async def test_chat_completion_agentic_hooks_move_past_a_plan_that_declines_the_loop(): + override: Final = ModelResponse() + declined: Final = _PlannedChatLoop(AgenticLoopPlan(run_agentic_loop=False)) + overriding: Final = _PlannedChatLoop(AgenticLoopPlan(response_override=override)) + + assert await _run_chat_completion_agentic_hooks(ModelResponse(), declined, overriding) is override + + +@pytest.mark.asyncio +async def test_chat_completion_agentic_hooks_return_a_run_only_loops_result(): + loop_result: Final = ModelResponse() + + assert await _run_chat_completion_agentic_hooks(ModelResponse(), _RunOnlyChatLoop(loop_result)) is loop_result + + @pytest.mark.asyncio async def test_chat_completion_agentic_followup_does_not_repeat_request_params_from_plan_kwargs(monkeypatch): """A plan whose kwargs repeat a request param, or the explicitly passed model, must not crash the chat follow-up with a duplicate keyword""" From b3849b29de0e119d90685020a48504388747c152 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 09:58:35 -0700 Subject: [PATCH 02/12] fix(agentic): surface failed follow-ups from the OpenAI chat dispatcher and update pinned integration tests --- litellm/llms/openai/openai.py | 64 ++++++-------- ...t_interception_converted_stream_options.py | 64 ++++---------- tests/unit/llms/openai/test_openai.py | 86 +++++++++++++++++++ 3 files changed, 128 insertions(+), 86 deletions(-) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 9d4df354a30..ee3dad39ce5 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -647,48 +647,34 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): custom_llm_provider: Final = litellm_params.get("custom_llm_provider", "openai") for callback in callbacks: + if not isinstance(callback, CustomLogger): + continue try: - if isinstance(callback, CustomLogger): - # Check if the callback has the chat completion agentic loop methods - if not hasattr(callback, "async_should_run_chat_completion_agentic_loop"): - continue - - # First: Check if agentic loop should run (using chat completion method) - ( - should_run, - tool_calls, - ) = await callback.async_should_run_chat_completion_agentic_loop( - response=response, - model=model, - messages=messages, - tools=tools, - stream=stream, - custom_llm_provider=custom_llm_provider, - kwargs=litellm_params, - ) - - if should_run: - # Second: Execute agentic loop - kwargs_with_provider = litellm_params.copy() if litellm_params else {} - kwargs_with_provider["custom_llm_provider"] = custom_llm_provider - - # For OpenAI Chat Completions, use the chat completion agentic loop method - agentic_response: object = await callback.async_run_chat_completion_agentic_loop( - tools=tool_calls, - model=model, - messages=messages, - response=response, - optional_params=optional_params, - logging_obj=logging_obj, - stream=stream, - kwargs=kwargs_with_provider, - ) - # First hook that runs agentic loop wins - return agentic_response - + should_run, tool_calls = await callback.async_should_run_chat_completion_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=stream, + custom_llm_provider=custom_llm_provider, + kwargs=litellm_params, + ) except Exception as e: verbose_logger.exception( - "LiteLLM.AgenticHookError: Exception in agentic completion hooks for OpenAI: %s", e + "LiteLLM.AgenticHookError: Exception in OpenAI chat completion agentic gate: %s", e + ) + continue + + if should_run: + return await callback.async_run_chat_completion_agentic_loop( + tools=tool_calls, + model=model, + messages=messages, + response=response, + optional_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs={**(litellm_params or {}), "custom_llm_provider": custom_llm_provider}, ) return None diff --git a/tests/integration/streaming/test_interception_converted_stream_options.py b/tests/integration/streaming/test_interception_converted_stream_options.py index 0ca62d2f014..85b65683ba6 100644 --- a/tests/integration/streaming/test_interception_converted_stream_options.py +++ b/tests/integration/streaming/test_interception_converted_stream_options.py @@ -754,60 +754,30 @@ def test_wrong_azure_key_surfaces_azures_401(rig: _Rig, scenario: Scenario) -> N assert len(_upstream(rig, deployment).model_calls) == 1 -def _tool_call(call: Mapping[str, JsonValue]) -> tuple[str, Mapping[str, JsonValue]]: - function: Final = object_value(call["function"]) - return string_value(function["name"]), iv.JSON_OBJECT.validate_json(string_value(function["arguments"])) +def _assert_failed_once(call_id: str) -> None: + rows: Final = eventually( + lambda: read_rows('SELECT status FROM "LiteLLM_SpendLogs" WHERE litellm_call_id = %s', (call_id,)), + lambda found: len(found) >= 1, + seconds=70, + ) + assert [row["status"] for row in rows] == ["failure"], rows -def _stream_choices(stream: _Stream) -> Iterator[Mapping[str, JsonValue]]: - for frame in stream.frames: - yield from rv.ITEMS.validate_python(frame.get("choices") or []) - - -def _streamed_tool_calls(stream: _Stream) -> Iterator[tuple[str, Mapping[str, JsonValue]]]: - for choice in _stream_choices(stream): - yield from map( - _tool_call, rv.ITEMS.validate_python(object_value(choice.get("delta") or {}).get("tool_calls") or []) - ) - - -def _handed_back(deployment: _Deployment) -> list[tuple[str, Mapping[str, JsonValue]]]: - return [(iv.SEARCH_TOOL, {"query": iv.query(deployment.name)})] - - -def _assert_succeeded_once(model: str, identity: str) -> None: - rows: Final = _spend_rows(model, 1) - assert [string_value(row["request_id"]) for row in rows if row["status"] == "success"] == [identity], rows - - -def test_failed_follow_up_call_hands_the_search_tool_call_back_with_its_usage(rig: _Rig, scenario: Scenario) -> None: +@pytest.mark.parametrize("stream", [True, False], ids=["stream", "json"]) +def test_failed_follow_up_call_answers_with_the_providers_error_instead_of_the_search_tool_call( + rig: _Rig, scenario: Scenario, stream: bool +) -> None: deployment: Final = _deploy(rig, scenario, "followup-fails") - stream: Final = _chat(rig, _chat_body(deployment, extra={**_NO_CACHE, "stream_options": dict(_USAGE)})) - assert stream.status == 200 and stream.content == "", stream.text - assert list(_streamed_tool_calls(stream)) == _handed_back(deployment), stream.text - finished: Final = [choice["finish_reason"] for choice in _stream_choices(stream) if choice.get("finish_reason")] - assert finished == ["tool_calls"], stream.text - assert [_totals(usage) for usage in stream.usages] == [iv.FIRST_USAGE["total_tokens"]], stream.text - (identity,) = stream.ids + extra: Final = {**_NO_CACHE, "stream_options": dict(_USAGE)} if stream else _NO_CACHE + body: Final = _chat_body(deployment, stream=stream, extra=extra) + response: Final = rig.gateway.request("POST", "/v1/chat/completions", body) + assert response.status_code == 500 and iv.FOLLOWUP_FAILURE in response.text, response.text + assert iv.SEARCH_TOOL not in response.text, response.text _assert_upstream(_upstream(rig, deployment), model_calls=2, searches=1) - _assert_succeeded_once(deployment.model, identity) + _assert_failed_once(response.headers["x-litellm-call-id"]) assert rig.gateway.chat(_deploy(rig, scenario, "plain").model)["object"] == "chat.completion" -def test_failed_follow_up_call_on_a_json_request_hands_the_search_tool_call_back(rig: _Rig, scenario: Scenario) -> None: - deployment: Final = _deploy(rig, scenario, "followup-fails") - response: Final = rig.gateway.request("POST", "/v1/chat/completions", _chat_body(deployment, stream=False)) - assert response.status_code == 200, response.text - body: Final = iv.JSON_OBJECT.validate_json(response.content) - (choice,) = rv.ITEMS.validate_python(body["choices"]) - calls: Final = rv.ITEMS.validate_python(object_value(choice["message"])["tool_calls"]) - assert [_tool_call(call) for call in calls] == _handed_back(deployment), body - assert choice["finish_reason"] == "tool_calls", body - assert _totals(object_value(body["usage"])) == iv.FIRST_USAGE["total_tokens"], body - _assert_upstream(_upstream(rig, deployment), model_calls=2, searches=1) - _assert_succeeded_once(deployment.model, string_value(body["id"])) - - def test_search_outage_still_restreams_the_follow_up_answer(rig: _Rig, scenario: Scenario) -> None: deployment: Final = _deploy(rig, scenario, "search-down") stream: Final = _chat(rig, _chat_body(deployment, extra={**_NO_CACHE, "stream_options": dict(_USAGE)})) diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 11d7c7d6d3c..b1a62a0de63 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -18,6 +18,7 @@ from pydantic import BaseModel, ConfigDict, JsonValue import litellm from litellm import create_thread, get_thread from litellm.caching.caching import Cache, LiteLLMCacheType +from litellm.integrations.custom_logger import CustomLogger from litellm.llms.openai.openai import ( AssistantEventHandler, AsyncAssistantEventHandler, @@ -222,6 +223,91 @@ async def test_acompletion_returns_prompt_cache_details_over_injected_transport( assert prompt_tokens_details.cached_tokens == 7 + +class _FailingFollowUpChatLoop(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.runs = 0 + + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: list[dict], + tools: list[dict] | None, + stream: bool, + custom_llm_provider: str, + kwargs: dict, + ) -> tuple[bool, dict]: + return True, {"tool_calls": [{"id": "call-1"}]} + + async def async_run_chat_completion_agentic_loop( + self, + tools: dict, + model: str, + messages: list[dict], + response: object, + optional_params: dict, + logging_obj: object, + stream: bool, + kwargs: dict, + ) -> object: + self.runs += 1 + raise litellm.RateLimitError(message="follow-up rate limited", llm_provider="openai", model=model) + + +@pytest.mark.asyncio +async def test_acompletion_surfaces_a_failed_agentic_follow_up_instead_of_the_first_turn( + monkeypatch: pytest.MonkeyPatch, +): + loop: Final = _FailingFollowUpChatLoop() + monkeypatch.setattr(litellm, "callbacks", [loop]) + + def respond(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-search", + "object": "chat.completion", + "created": 0, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "litellm_web_search", "arguments": '{"query": "weather"}'}, + } + ], + }, + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, + }, + ) + + async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + with pytest.raises(litellm.RateLimitError, match="follow-up rate limited"): + await asyncio.wait_for( + litellm.acompletion( + model="openai/gpt-5.6", + api_key="transport-only", + client=AsyncOpenAI(api_key="transport-only", http_client=http_client), + messages=[{"role": "user", "content": "search the web for the weather"}], + num_retries=0, + max_retries=0, + ), + timeout=10, + ) + + assert loop.runs == 1 + _REASONING_TOOL_CALL_REPLY: Final = { "id": "chatcmpl-fake-stream", "object": "chat.completion", From e9ce4956285fc0d6894379e6dcce5c07026f07e5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:43:00 -0700 Subject: [PATCH 03/12] fix(agentic): end a capped agentic turn on chat and responses instead of a retried 500 --- .../agentic_loop_refusal.py | 72 +++++++++ .../chat_completion_agentic_loop.py | 53 +++++-- litellm/llms/custom_httpx/llm_http_handler.py | 42 ++--- litellm/llms/openai/openai.py | 5 +- .../test_websearch_agentic_loop_cap.py | 95 ++++++++++-- .../test_chat_completion_agentic_loop.py | 143 +++++++++++------- tests/unit/llms/openai/test_openai.py | 134 +++++++++++----- 7 files changed, 390 insertions(+), 154 deletions(-) create mode 100644 litellm/litellm_core_utils/agentic_loop_refusal.py diff --git a/litellm/litellm_core_utils/agentic_loop_refusal.py b/litellm/litellm_core_utils/agentic_loop_refusal.py new file mode 100644 index 00000000000..420a31fd516 --- /dev/null +++ b/litellm/litellm_core_utils/agentic_loop_refusal.py @@ -0,0 +1,72 @@ +from collections.abc import Iterable +from itertools import chain +from typing import Final + +from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of +from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import Choices, ModelResponse + + +def _strings(values: Iterable[object]) -> frozenset[str]: + return frozenset(value for value in values if isinstance(value, str)) + + +def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]: + wrapped: Final = message_field(tool_calls, "tool_calls") + calls: Final = parts_of(tool_calls if wrapped is None else wrapped) + ids: Final = _strings( + chain.from_iterable((message_field(call, "id"), message_field(call, "call_id")) for call in calls) + ) + names: Final = _strings(message_field(call, "name") for call in calls) + return ids, names + + +def is_refused_call(call_id: object, name: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool: + if isinstance(call_id, str) and refused_ids: + return call_id in refused_ids + return name in refused_names + + +def is_refused_item( + item: object, item_type: str, id_field: str, refused_ids: frozenset[str], refused_names: frozenset[str] +) -> bool: + if message_field(item, "type") != item_type: + return False + return is_refused_call(message_field(item, id_field), message_field(item, "name"), refused_ids, refused_names) + + +def finalize_refused_responses_turn(response: ResponsesAPIResponse, tool_calls: object) -> ResponsesAPIResponse: + refused_ids, refused_names = refused_tool_call_identifiers(tool_calls) + kept: Final = [ + item + for item in parts_of(message_field(response, "output")) + if not is_refused_item(item, "function_call", "call_id", refused_ids, refused_names) + ] + return response.model_copy(update={"output": kept}) + + +def _is_refused_chat_tool_call(call: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool: + name: Final = message_field(message_field(call, "function"), "name") + return is_refused_call(message_field(call, "id"), name, refused_ids, refused_names) + + +def _finalized_choice(choice: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> object: + if not isinstance(choice, Choices) or not choice.message.tool_calls: + return choice + kept: Final = [ + call for call in choice.message.tool_calls if not _is_refused_chat_tool_call(call, refused_ids, refused_names) + ] + finish_reason: Final = "stop" if not kept and choice.finish_reason == "tool_calls" else choice.finish_reason + return choice.model_copy( + update={ + "message": choice.message.model_copy(update={"tool_calls": kept or None}), + "finish_reason": finish_reason, + } + ) + + +def finalize_refused_chat_turn(response: ModelResponse, tool_calls: object) -> ModelResponse: + refused_ids, refused_names = refused_tool_call_identifiers(tool_calls) + return response.model_copy( + update={"choices": [_finalized_choice(choice, refused_ids, refused_names) for choice in response.choices]} + ) diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index a43bb6c07b9..b094010fc14 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -11,6 +11,7 @@ from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs +from litellm.litellm_core_utils.agentic_loop_refusal import finalize_refused_chat_turn from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, @@ -22,6 +23,7 @@ from litellm.types.integrations.custom_logger import ( NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, AgenticLoopPlan, AgenticLoopRequestPatch, + AgenticLoopSafetyError, converted_stream_requested, is_interception_internal_key, stashed_stream_options, @@ -101,9 +103,9 @@ def _check_agentic_loop_safety( ) -> str: fingerprint: Final = _fingerprint_tools(tool_calls) if fingerprint in fingerprints: - raise ValueError("Agentic loop detected repeated tool-call fingerprint; aborting rerun") + raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun") if depth >= max_loops: - raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}") + raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}") return fingerprint @@ -132,6 +134,26 @@ def _converted_stream_replay( ) +def _end_refused_turn( + finalized: ModelResponse, + *, + kwargs: Mapping[str, object], + depth: int, + model: str, + custom_llm_provider: str, + logging_obj: object, +) -> ModelResponse | CustomStreamWrapper: + replay: Final = _converted_stream_replay( + finalized, + kwargs=kwargs, + depth=depth, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + return finalized if replay is None else replay + + def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]: metadata: Final = kwargs_for_followup.get("litellm_metadata") return MappingProxyType( @@ -296,13 +318,26 @@ async def maybe_run_chat_completion_agentic_loop( if not should_run: continue - fingerprint = _check_agentic_loop_safety( - tool_calls=tool_calls, - fingerprints=fingerprints, - depth=depth, - max_loops=max_loops, - model=model, - ) + try: + fingerprint = _check_agentic_loop_safety( + tool_calls=tool_calls, + fingerprints=fingerprints, + depth=depth, + max_loops=max_loops, + model=model, + ) + except AgenticLoopSafetyError as e: + if stream: + raise + verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [model=%s]: %s", model, str(e)) + return _end_refused_turn( + finalize_refused_chat_turn(response, tool_calls), + kwargs=kwargs, + depth=depth, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) if not _build_plan_overridden(callback): return await callback.async_run_agentic_loop( diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index d659bd75829..4f7792ec4d6 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -41,6 +41,11 @@ from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, up from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.files.types import FileContentStreamingResult from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs +from litellm.litellm_core_utils.agentic_loop_refusal import ( + finalize_refused_responses_turn, + is_refused_item, + refused_tool_call_identifiers, +) from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, @@ -5864,26 +5869,6 @@ class BaseLLMHTTPHandler: except Exception: return str(tools) - @staticmethod - def _refused_agentic_tool_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]: - """ - Collect the ids and names of the tool calls a safety rail just refused. - - Callbacks hand back either a bare list of tool calls or a dict wrapping - that list under ``tool_calls``, and both the anthropic and responses - shapes carry an ``id`` (or ``call_id``) plus a ``name``. - """ - calls: Final = tool_calls.get("tool_calls") if isinstance(tool_calls, dict) else tool_calls - if not isinstance(calls, list): - return frozenset(), frozenset() - dict_calls: Final = (call for call in calls if isinstance(call, dict)) - fields: Final = tuple((call.get("id"), call.get("call_id"), call.get("name")) for call in dict_calls) - ids: Final = frozenset( - value for call_id, caller_id, _ in fields for value in (call_id, caller_id) if isinstance(value, str) - ) - names: Final = frozenset(name for _, _, name in fields if isinstance(name, str)) - return ids, names - @staticmethod def _is_refused_tool_use_block(block: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool: """ @@ -5894,22 +5879,13 @@ class BaseLLMHTTPHandler: share a name with a refused one. The name is only consulted for tool call shapes that arrive without an id. """ - if not isinstance(block, dict) or block.get("type") != "tool_use": - return False - block_id: Final = block.get("id") - if isinstance(block_id, str) and refused_ids: - return block_id in refused_ids - return block.get("name") in refused_names + return is_refused_item(block, "tool_use", "id", refused_ids, refused_names) @staticmethod def _can_replace_turn_with_terminal_response(stream: bool, api_surface: str) -> bool: """ Whether a refused rerun can still be answered with a finalized turn. - Only the anthropic messages surface can. The responses surface carries a - pydantic model the finalizer does not rewrite, so it keeps raising, which - is what every surface did before this path learned to end the turn. - The messages and responses call sites pass ``stream=False``, because interception converts an intercepted stream to non-streaming before the loop runs and rebuilds the SSE stream from the finalized turn @@ -5918,7 +5894,7 @@ class BaseLLMHTTPHandler: finalized turn would reach the client as a second message rather than as a replacement. """ - return not stream and api_surface == "anthropic_messages" + return not stream and api_surface in ("anthropic_messages", "responses") @staticmethod def _finalize_refused_agentic_response(response: object, tool_calls: object) -> object: @@ -5935,10 +5911,12 @@ class BaseLLMHTTPHandler: one is still in the response the stop reason stays ``tool_use`` so the client knows to answer it. """ + if isinstance(response, ResponsesAPIResponse): + return finalize_refused_responses_turn(response, tool_calls) if not isinstance(response, dict): return response - refused_ids, refused_names = BaseLLMHTTPHandler._refused_agentic_tool_identifiers(tool_calls) + refused_ids, refused_names = refused_tool_call_identifiers(tool_calls) finalized: Final = dict(response) content: Final = finalized.get("content") if isinstance(content, list): diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index ee3dad39ce5..0d72dd01854 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -646,9 +646,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): # Get custom_llm_provider from litellm_params custom_llm_provider: Final = litellm_params.get("custom_llm_provider", "openai") - for callback in callbacks: - if not isinstance(callback, CustomLogger): - continue + custom_loggers: Final = tuple(callback for callback in callbacks if isinstance(callback, CustomLogger)) + for callback in custom_loggers: try: should_run, tool_calls = await callback.async_should_run_chat_completion_agentic_loop( response=response, diff --git a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py index 4449c7e7013..8cc44f1ea1c 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py @@ -31,6 +31,7 @@ from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.secret_managers.main import get_secret +from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.integrations.custom_logger import ( AgenticLoopPlan, AgenticLoopRequestPatch, @@ -99,6 +100,29 @@ def _tool_use_names(response: dict) -> list[str]: return [block.get("name") for block in response["content"] if block.get("type") == "tool_use"] +def _responses_turn(*function_calls: tuple[str, str]) -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id="resp_1", + created_at=0, + status="completed", + model="claude-sonnet-4-5", + object="response", + output=[ + { + "type": "message", + "id": "msg_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "Let me check one more source.", "annotations": []}], + }, + *( + {"type": "function_call", "id": f"fc_{call_id}", "call_id": call_id, "name": name, "arguments": "{}"} + for call_id, name in function_calls + ), + ], + ) + + class _InterceptingCallback(CustomLogger): """ Stands in for the websearch interceptor: asks for another loop whenever the @@ -152,6 +176,18 @@ class _InterceptingCallback(CustomLogger): return response +class _ResponsesInterceptingCallback(_InterceptingCallback): + async def async_should_run_agentic_loop( + self, response, model, messages, tools, stream, custom_llm_provider, kwargs + ): + calls = [ + {"id": item.call_id, "call_id": item.call_id, "name": item.name} + for item in response.output + if item.type == "function_call" and item.name == INTERNAL_TOOL_NAME + ] + return bool(calls), {"tool_calls": calls} + + def _logging_obj(callback: CustomLogger, converted_stream: bool = False) -> MagicMock: logging_obj = MagicMock() logging_obj.model_call_details = {"websearch_interception_converted_stream": converted_stream} @@ -347,6 +383,23 @@ class TestCappedLoopReturnsTerminalResponse: assert _tool_use_names(result) == [] assert result["stop_reason"] == "end_turn" + def test_a_name_only_refusal_keeps_the_native_search_blocks_sharing_that_name(self): + """ + The gate also picks up a bare web_search tool_use, and the native + server_tool_use blocks already in the turn carry that same name. Only the + tool_use goes, so each web_search_tool_result keeps the block it answers. + """ + response = _response_asking_for_another_search() + response["content"][-1] = {"type": "tool_use", "name": "web_search", "input": {}} + + result = BaseLLMHTTPHandler._finalize_refused_agentic_response( + response=response, + tool_calls={"tool_calls": [{"name": "web_search", "input": {}}]}, + ) + + assert _block_types(result) == ["server_tool_use", "web_search_tool_result", "text"] + assert result["stop_reason"] == "end_turn" + @pytest.mark.asyncio async def test_streaming_caller_is_left_to_its_existing_behavior(self): """ @@ -365,19 +418,39 @@ class TestCappedLoopReturnsTerminalResponse: assert self.callback.plan_calls == 0 @pytest.mark.asyncio - async def test_responses_surface_is_left_to_its_existing_behavior(self): + async def test_responses_surface_drops_the_refused_function_call_and_keeps_the_answer(self): + response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_client", "lookup")) + callback = _ResponsesInterceptingCallback() + + result = await _run_hooks( + self.handler, + callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + response=response, + api_surface="responses", + ) + + assert isinstance(result, ResponsesAPIResponse) + assert [(item.type, getattr(item, "name", None)) for item in result.output] == [ + ("message", None), + ("function_call", "lookup"), + ] + assert callback.plan_calls == 0 + + def test_responses_refusal_by_call_id_alone_keeps_the_calls_it_did_not_list(self): """ - The responses surface carries a pydantic model rather than the anthropic - dict this finalizer rewrites, so it keeps raising instead of being handed - a response that was never actually finalized. + The code interpreter gate hands over call_id and name with no id, so the + call_id is what scopes the refusal there, the same way the id does on + /v1/messages. """ - with pytest.raises(AgenticLoopSafetyError): - await _run_hooks( - self.handler, - self.callback, - kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, - api_surface="responses", - ) + response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_unlisted", INTERNAL_TOOL_NAME)) + + result = BaseLLMHTTPHandler._finalize_refused_agentic_response( + response=response, + tool_calls={"tool_calls": [{"call_id": "call_search", "name": INTERNAL_TOOL_NAME}]}, + ) + + assert [getattr(item, "call_id", None) for item in result.output] == [None, "call_unlisted"] @pytest.mark.asyncio async def test_non_dict_response_is_returned_untouched(self): diff --git a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py index 4551c0d22fb..acb3987341f 100644 --- a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -21,6 +21,7 @@ they stay out of the body even without it. """ import datetime +import json from typing import Any, Dict, Final, List, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch @@ -43,6 +44,7 @@ from litellm.types.integrations.custom_logger import ( WEBSEARCH_INTERCEPTION_PREFIX, AgenticLoopPlan, AgenticLoopRequestPatch, + AgenticLoopSafetyError, ) from litellm.types.utils import ( ChatCompletionMessageToolCall, @@ -398,72 +400,97 @@ async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_requ assert call_kwargs["api_base"] == "https://a" +_REFUSED_GATE_TOOL_CALLS: Final = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]} +_REFUSING_RAILS: Final = ( + {"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + { + "_agentic_loop_depth": 0, + "max_agentic_loops": 3, + "_agentic_loop_fingerprints": [json.dumps(_REFUSED_GATE_TOOL_CALLS, sort_keys=True, default=str)], + }, +) + + +def _tool_call(call_id: str, name: str) -> ChatCompletionMessageToolCall: + return ChatCompletionMessageToolCall(id=call_id, type="function", function=Function(name=name, arguments="{}")) + + +async def _run_refused_rerun( + response: ModelResponse, + rail_kwargs: Dict[str, object], + stream: bool = False, + gate_tool_calls: Optional[Dict[str, object]] = _REFUSED_GATE_TOOL_CALLS, +) -> Any: + litellm.callbacks = [_GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=True), tool_calls=gate_tool_calls)] + acompletion_mock = AsyncMock() + with patch.object(litellm, "acompletion", acompletion_mock): + try: + return await maybe_run_chat_completion_agentic_loop( + response=response, + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + kwargs=dict(rail_kwargs), + logging_obj=_LoggingStub(), + custom_llm_provider="openai", + stream=stream, + ) + finally: + acompletion_mock.assert_not_awaited() + + @pytest.mark.asyncio -async def test_dispatcher_raises_when_depth_reaches_max_agentic_loops( - restore_callbacks, +@pytest.mark.parametrize("rail_kwargs", _REFUSING_RAILS, ids=["max_agentic_loops", "repeated_fingerprint"]) +async def test_refused_rerun_ends_the_turn_without_the_internal_tool_call( + restore_callbacks, rail_kwargs: Dict[str, object] ): - """depth >= max_agentic_loops -> ValueError mentioning max_agentic_loops, - before any follow-up call is attempted.""" - logger = _GateOnlyLogger( - plan=AgenticLoopPlan(run_agentic_loop=True), - tool_calls={"tool_calls": [{"id": "call_abc"}]}, - ) - litellm.callbacks = [logger] + result = await _run_refused_rerun(_tool_call_model_response(), rail_kwargs) - acompletion_mock = AsyncMock() - with patch.object(litellm, "acompletion", acompletion_mock): - with pytest.raises(ValueError, match="max_agentic_loops"): - await maybe_run_chat_completion_agentic_loop( - response=_tool_call_model_response(), - model="gpt-4o-mini", - messages=[{"role": "user", "content": "hi"}], - optional_params={}, - kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, - logging_obj=_LoggingStub(), - custom_llm_provider="openai", - stream=False, - ) - - acompletion_mock.assert_not_awaited() + assert isinstance(result, ModelResponse) + assert not result.choices[0].message.tool_calls + assert result.choices[0].finish_reason == "stop" @pytest.mark.asyncio -async def test_dispatcher_raises_on_repeated_tool_call_fingerprint(restore_callbacks): - """A tool_calls fingerprint already present in _agentic_loop_fingerprints -> - ValueError about the repeated fingerprint (cycle guard), with no follow-up - call.""" - import json - - # The dispatcher fingerprints the whole value the gate returns as its second - # tuple element, so the seeded fingerprint must mirror that dict exactly. - gate_tool_calls = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]} - fingerprint = json.dumps(gate_tool_calls, sort_keys=True, default=str) - - logger = _GateOnlyLogger( - plan=AgenticLoopPlan(run_agentic_loop=True), - tool_calls=gate_tool_calls, - ) - litellm.callbacks = [logger] - - acompletion_mock = AsyncMock() - with patch.object(litellm, "acompletion", acompletion_mock): - with pytest.raises(ValueError, match="fingerprint"): - await maybe_run_chat_completion_agentic_loop( - response=_tool_call_model_response(), - model="gpt-4o-mini", - messages=[{"role": "user", "content": "hi"}], - optional_params={}, - kwargs={ - "_agentic_loop_depth": 0, - "max_agentic_loops": 3, - "_agentic_loop_fingerprints": [fingerprint], - }, - logging_obj=_LoggingStub(), - custom_llm_provider="openai", - stream=False, +async def test_refused_rerun_keeps_the_clients_own_tool_call(restore_callbacks): + response = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message=Message( + role="assistant", + content="Running it.", + tool_calls=[_tool_call("call_abc", "litellm_code_execution"), _tool_call("call_client", "lookup")], + ), ) + ] + ) - acompletion_mock.assert_not_awaited() + result = await _run_refused_rerun(response, _REFUSING_RAILS[0]) + + assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"] + assert result.choices[0].finish_reason == "tool_calls" + assert result.choices[0].message.content == "Running it." + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "response, gate_tool_calls", + [(_plain_model_response(), _REFUSED_GATE_TOOL_CALLS), (_tool_call_model_response(), None)], + ids=["turn_has_no_tool_calls", "gate_names_no_tool_call"], +) +async def test_refused_rerun_returns_a_turn_it_cannot_match_unchanged( + restore_callbacks, response: ModelResponse, gate_tool_calls: Optional[Dict[str, object]] +): + result = await _run_refused_rerun(response, _REFUSING_RAILS[0], gate_tool_calls=gate_tool_calls) + + assert result.choices[0].model_dump() == response.choices[0].model_dump() + + +@pytest.mark.asyncio +async def test_refused_rerun_on_a_live_stream_still_raises(restore_callbacks): + with pytest.raises(AgenticLoopSafetyError, match="max_agentic_loops"): + await _run_refused_rerun(_tool_call_model_response(), _REFUSING_RAILS[0], stream=True) @pytest.mark.asyncio diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index b1a62a0de63..c2d51df48d0 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -256,6 +256,84 @@ class _FailingFollowUpChatLoop(CustomLogger): raise litellm.RateLimitError(message="follow-up rate limited", llm_provider="openai", model=model) +class _RaisingAgenticGate(CustomLogger): + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: list[dict], + tools: list[dict] | None, + stream: bool, + custom_llm_provider: str, + kwargs: dict, + ) -> tuple[bool, dict]: + raise ValueError("gate bug") + + +class _AnsweringChatLoop(_FailingFollowUpChatLoop): + async def async_run_chat_completion_agentic_loop( + self, + tools: dict, + model: str, + messages: list[dict], + response: object, + optional_params: dict, + logging_obj: object, + stream: bool, + kwargs: dict, + ) -> object: + self.runs += 1 + return litellm.ModelResponse( + choices=[ + {"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "after the search"}} + ] + ) + + +def _respond_with_a_search_tool_call(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": "chatcmpl-search", + "object": "chat.completion", + "created": 0, + "model": "gpt-5.6", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "call-1", + "type": "function", + "function": {"name": "litellm_web_search", "arguments": '{"query": "weather"}'}, + } + ], + }, + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, + }, + ) + + +async def _acompletion_through(http_client: httpx.AsyncClient) -> object: + return await asyncio.wait_for( + litellm.acompletion( + model="openai/gpt-5.6", + api_key="transport-only", + client=AsyncOpenAI(api_key="transport-only", http_client=http_client), + messages=[{"role": "user", "content": "search the web for the weather"}], + num_retries=0, + max_retries=0, + ), + timeout=10, + ) + + @pytest.mark.asyncio async def test_acompletion_surfaces_a_failed_agentic_follow_up_instead_of_the_first_turn( monkeypatch: pytest.MonkeyPatch, @@ -263,51 +341,25 @@ async def test_acompletion_surfaces_a_failed_agentic_follow_up_instead_of_the_fi loop: Final = _FailingFollowUpChatLoop() monkeypatch.setattr(litellm, "callbacks", [loop]) - def respond(request: httpx.Request) -> httpx.Response: - return httpx.Response( - 200, - json={ - "id": "chatcmpl-search", - "object": "chat.completion", - "created": 0, - "model": "gpt-5.6", - "choices": [ - { - "index": 0, - "finish_reason": "tool_calls", - "message": { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call-1", - "type": "function", - "function": {"name": "litellm_web_search", "arguments": '{"query": "weather"}'}, - } - ], - }, - } - ], - "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, - }, - ) - - async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client: + async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client: with pytest.raises(litellm.RateLimitError, match="follow-up rate limited"): - await asyncio.wait_for( - litellm.acompletion( - model="openai/gpt-5.6", - api_key="transport-only", - client=AsyncOpenAI(api_key="transport-only", http_client=http_client), - messages=[{"role": "user", "content": "search the web for the weather"}], - num_retries=0, - max_retries=0, - ), - timeout=10, - ) + await _acompletion_through(http_client) assert loop.runs == 1 + +@pytest.mark.asyncio +async def test_acompletion_skips_a_failing_agentic_gate_and_runs_the_next_hook(monkeypatch: pytest.MonkeyPatch): + loop: Final = _AnsweringChatLoop() + monkeypatch.setattr(litellm, "callbacks", [_RaisingAgenticGate(), loop]) + + async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client: + response = await _acompletion_through(http_client) + + assert loop.runs == 1 + assert response.choices[0].message.content == "after the search" + + _REASONING_TOOL_CALL_REPLY: Final = { "id": "chatcmpl-fake-stream", "object": "chat.completion", From e9429857d7cecd61d20ee8430a19a44a3d7a993c Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 29 Oct 2026 12:59:47 -0700 Subject: [PATCH 04/12] fix(openai): run agentic hooks after the first call's error handling so a follow-up error keeps its own type and message --- litellm/llms/openai/openai.py | 47 ++++---- .../test_websearch_agentic_loop_cap.py | 16 +++ tests/unit/llms/openai/test_openai.py | 107 +++++++++++++++++- 3 files changed, 141 insertions(+), 29 deletions(-) diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index 0d72dd01854..d36ae19b698 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -1024,30 +1024,6 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): hidden_params={"headers": headers}, _response_headers=headers, ) - - # Call agentic completion hooks (e.g., for websearch_interception) - agentic_response = await self._call_agentic_completion_hooks_openai( - response=final_response_obj, - model=model, - messages=messages, - optional_params=optional_params, - logging_obj=logging_obj, - stream=False, - litellm_params=litellm_params, - ) - - if agentic_response is not None: - final_response_obj = agentic_response - - if fake_stream is True: - return self.mock_streaming( - response=cast(ModelResponse, final_response_obj), - logging_obj=logging_obj, - model=model, - stream_options=stream_options, - ) - - return final_response_obj except openai.UnprocessableEntityError as e: ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: @@ -1070,6 +1046,29 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): headers=error_headers, body=exception_body, ) + else: + agentic_response = await self._call_agentic_completion_hooks_openai( + response=final_response_obj, + model=model, + messages=messages, + optional_params=optional_params, + logging_obj=logging_obj, + stream=False, + litellm_params=litellm_params, + ) + + if agentic_response is not None: + final_response_obj = agentic_response + + if fake_stream is True: + return self.mock_streaming( + response=cast(ModelResponse, final_response_obj), + logging_obj=logging_obj, + model=model, + stream_options=stream_options, + ) + + return final_response_obj def streaming( self, diff --git a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py index 8cc44f1ea1c..60d77fd6dda 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py @@ -625,6 +625,22 @@ class TestFailedFollowUpCallReachesTheClient: kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True}, ) + @pytest.mark.asyncio + async def test_responses_follow_up_rate_limit_is_raised_to_the_caller(self, monkeypatch): + monkeypatch.setattr(litellm, "aresponses", self._rate_limited_follow_up) + callback = _ResponsesInterceptingCallback() + + with pytest.raises(RateLimitError, match="UserByModelByMinuteUncachedInputTokens"): + await _run_hooks( + self.handler, + callback, + kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3}, + response=_responses_turn(("call_search", INTERNAL_TOOL_NAME)), + api_surface="responses", + ) + + assert callback.plan_calls == 1 + class TestFailedSearchEndsTheTurn: """ diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index c2d51df48d0..9aa5a06a006 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -224,7 +224,7 @@ async def test_acompletion_returns_prompt_cache_details_over_injected_transport( -class _FailingFollowUpChatLoop(CustomLogger): +class _SearchingChatGate(CustomLogger): def __init__(self) -> None: super().__init__() self.runs = 0 @@ -241,6 +241,12 @@ class _FailingFollowUpChatLoop(CustomLogger): ) -> tuple[bool, dict]: return True, {"tool_calls": [{"id": "call-1"}]} + +class _FailingFollowUpChatLoop(_SearchingChatGate): + def __init__(self, error: Exception) -> None: + super().__init__() + self.error = error + async def async_run_chat_completion_agentic_loop( self, tools: dict, @@ -253,7 +259,7 @@ class _FailingFollowUpChatLoop(CustomLogger): kwargs: dict, ) -> object: self.runs += 1 - raise litellm.RateLimitError(message="follow-up rate limited", llm_provider="openai", model=model) + raise self.error class _RaisingAgenticGate(CustomLogger): @@ -270,7 +276,7 @@ class _RaisingAgenticGate(CustomLogger): raise ValueError("gate bug") -class _AnsweringChatLoop(_FailingFollowUpChatLoop): +class _AnsweringChatLoop(_SearchingChatGate): async def async_run_chat_completion_agentic_loop( self, tools: dict, @@ -338,16 +344,107 @@ async def _acompletion_through(http_client: httpx.AsyncClient) -> object: async def test_acompletion_surfaces_a_failed_agentic_follow_up_instead_of_the_first_turn( monkeypatch: pytest.MonkeyPatch, ): - loop: Final = _FailingFollowUpChatLoop() + follow_up_error: Final = litellm.RateLimitError( + message="follow-up rate limited", llm_provider="openai", model="gpt-5.6" + ) + loop: Final = _FailingFollowUpChatLoop(follow_up_error) monkeypatch.setattr(litellm, "callbacks", [loop]) async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client: - with pytest.raises(litellm.RateLimitError, match="follow-up rate limited"): + with pytest.raises(litellm.RateLimitError) as raised: await _acompletion_through(http_client) + assert raised.value.message == follow_up_error.message assert loop.runs == 1 +@pytest.mark.asyncio +async def test_acompletion_does_not_resend_the_first_call_when_the_agentic_follow_up_answers_422( + monkeypatch: pytest.MonkeyPatch, +): + follow_up_error: Final = litellm.UnprocessableEntityError( + message="follow-up rejected", + model="gpt-5.6", + llm_provider="openai", + response=httpx.Response(422, request=httpx.Request("POST", "https://api.openai.com/v1/chat/completions")), + ) + loop: Final = _FailingFollowUpChatLoop(follow_up_error) + monkeypatch.setattr(litellm, "callbacks", [loop]) + monkeypatch.setattr(litellm, "drop_params", True) + + async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client: + with pytest.raises(litellm.UnprocessableEntityError) as raised: + await _acompletion_through(http_client) + + assert raised.value.message == follow_up_error.message + assert loop.runs == 1 + + +def _rejecting_the_user_param(outbound: asyncio.Queue[dict[str, object]]) -> httpx.MockTransport: + def respond(request: httpx.Request) -> httpx.Response: + body: Final = json.loads(request.content) + outbound.put_nowait(body) + if "user" in body: + rejection: Final = json.dumps({"detail": [{"loc": ["body", "user"], "msg": "extra fields not permitted"}]}) + return httpx.Response(422, json={"error": {"message": rejection}}) + return httpx.Response( + 200, + json={ + "id": "chatcmpl-resent", + "object": "chat.completion", + "created": 0, + "model": "gpt-5.6", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "done"}}], + }, + ) + + return httpx.MockTransport(respond) + + +async def _acompletion_with_a_user_param(http_client: httpx.AsyncClient) -> object: + return await asyncio.wait_for( + litellm.acompletion( + model="openai/gpt-5.6", + api_key="transport-only", + client=AsyncOpenAI(api_key="transport-only", http_client=http_client), + messages=[{"role": "user", "content": "hi"}], + user="end-user-1", + num_retries=0, + max_retries=0, + ), + timeout=10, + ) + + +@pytest.mark.asyncio +async def test_acompletion_resends_without_the_param_a_422_rejects_when_drop_params_is_set( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr(litellm, "drop_params", True) + outbound: Final[asyncio.Queue[dict[str, object]]] = asyncio.Queue() + + async with httpx.AsyncClient(transport=_rejecting_the_user_param(outbound)) as http_client: + response: Final = await _acompletion_with_a_user_param(http_client) + + assert response.choices[0].message.content == "done" + assert outbound.get_nowait()["user"] == "end-user-1" + assert "user" not in outbound.get_nowait() + assert outbound.empty() + + +@pytest.mark.asyncio +async def test_acompletion_raises_a_422_without_resending_when_drop_params_is_off(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "drop_params", False) + outbound: Final[asyncio.Queue[dict[str, object]]] = asyncio.Queue() + + async with httpx.AsyncClient(transport=_rejecting_the_user_param(outbound)) as http_client: + with pytest.raises(litellm.BadRequestError, match="extra fields not permitted"): + await _acompletion_with_a_user_param(http_client) + + assert outbound.get_nowait()["user"] == "end-user-1" + assert outbound.empty() + + @pytest.mark.asyncio async def test_acompletion_skips_a_failing_agentic_gate_and_runs_the_next_hook(monkeypatch: pytest.MonkeyPatch): loop: Final = _AnsweringChatLoop() From e52142072257e6b8e404fe6948a884bf142f1a06 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 12:04:15 -0700 Subject: [PATCH 05/12] fix(agentic loop): end capped turns on every chat path and bill the first call when a follow-up fails A turn refused at max_agentic_loops now drops the internal tool calls of every hook that claims calls in it, while a call the client declared survives. The hosted_vllm and openai chat paths now carry the loop depth, fingerprints and max_agentic_loops into the follow-up, so they stop at the cap instead of looping forever. A failed follow-up call now records the first call's usage and cost on the request's failure log before the error reaches the client. A code interpreter sandbox that is missing or fails becomes the tool call's output for the model to handle, so the turn completes without a retry. On native streaming /v1/messages a hook or follow-up failure now reaches the client as an Anthropic error event with the mapped type, and compresr declares its retrieve tool so the stream holds it back. Comments and docstrings the branch added against the repo rule are removed --- .../code_interpreter_interception/handler.py | 73 ++++- .../agentic_loop_refusal.py | 29 +- .../chat_completion_agentic_loop.py | 224 +++++++++++++-- .../litellm_core_utils/get_litellm_params.py | 4 + litellm/litellm_core_utils/litellm_logging.py | 5 + .../native/agentic_streaming_iterator.py | 10 +- litellm/llms/custom_httpx/llm_http_handler.py | 265 ++++++------------ litellm/llms/openai/openai.py | 67 ++--- litellm/main.py | 46 +-- .../guardrail_hooks/compresr/compresr.py | 4 +- ...t_interception_converted_stream_options.py | 25 +- .../test_handler.py | 76 ++++- .../test_websearch_agentic_loop_cap.py | 158 +++++++++-- .../test_chat_completion_agentic_loop.py | 91 +++++- .../native/test_agentic_streaming_iterator.py | 64 ++++- .../custom_httpx/test_llm_http_handler.py | 230 +++++++++++++-- tests/unit/llms/openai/test_openai.py | 94 +++++++ 17 files changed, 1109 insertions(+), 356 deletions(-) diff --git a/litellm/integrations/code_interpreter_interception/handler.py b/litellm/integrations/code_interpreter_interception/handler.py index 69b528a29f8..2cc925707ca 100644 --- a/litellm/integrations/code_interpreter_interception/handler.py +++ b/litellm/integrations/code_interpreter_interception/handler.py @@ -10,6 +10,7 @@ import json import time import uuid from collections.abc import Mapping, Sequence +from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypedDict, runtime_checkable from pydantic import ValidationError @@ -141,6 +142,31 @@ class _SupportsOutput(Protocol): _CachedContainer: TypeAlias = tuple[ContainerHandle, SandboxToolParams | None, float, str | None] +@dataclass(frozen=True, slots=True) +class _ReadySandbox: + container: ContainerHandle + params: SandboxToolParams | None + + +@dataclass(frozen=True, slots=True) +class _SandboxUnavailable: + reason: str + + +_Sandbox: TypeAlias = _ReadySandbox | _SandboxUnavailable + + +def _unavailable_output(unavailable: _SandboxUnavailable) -> str: + return f"[sandbox unavailable] {unavailable.reason}" + + +def _sandbox_unavailable(step: str, error: Exception) -> _SandboxUnavailable: + verbose_logger.warning( + "CodeInterpreterInterception: %s failed, answering the tool call with the failure: %s", step, error + ) + return _SandboxUnavailable(reason=str(error)) + + def _output_item_type(item: object) -> object: if isinstance(item, dict): item_mapping: Final[dict[str, object]] = item @@ -451,16 +477,16 @@ class CodeInterpreterInterceptionLogger(CustomLogger): sandbox_key: Final = self._extract_sandbox_key(kwargs) is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY)) identity: Final = _extract_identity(kwargs) if is_session else None - container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity) + sandbox: Final = await self._acquire_sandbox(cache_key=sandbox_key, identity=identity) try: - container_id: Final = self._container_id(container) + container_id: Final = self._sandbox_container_id(sandbox) input_list: Final = self._normalize_messages(messages) code_interpreter_calls: Final[list[CodeInterpreterCall]] = [] for tool_call in tool_calls: arguments = tool_call.get("arguments", "") code = self._parse_code(arguments) - stdout = await self._run_tool_call(container=container, params=params, arguments=arguments) + stdout = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments) input_list.append( { "type": "function_call", @@ -526,14 +552,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger): sandbox_key: Final = self._extract_sandbox_key(kwargs) is_session: Final = bool(kwargs.get(_SESSION_SCOPED_KEY)) identity: Final = _extract_identity(kwargs) if is_session else None - container, params = await self._get_or_create_container(cache_key=sandbox_key, identity=identity) + sandbox: Final = await self._acquire_sandbox(cache_key=sandbox_key, identity=identity) try: - container_id: Final = self._container_id(container) + container_id: Final = self._sandbox_container_id(sandbox) tool_results: Final = [ await self._build_chat_completion_tool_result( - container=container, - params=params, + sandbox=sandbox, tool_call=tool_call, container_id=container_id, ) @@ -569,9 +594,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger): ) @staticmethod - def _container_id(container: ContainerHandle) -> str | None: - container_id: Final[object] = getattr(container, "id", None) - return container_id if isinstance(container_id, str) else None + def _sandbox_container_id(sandbox: _Sandbox) -> str | None: + match sandbox: + case _ReadySandbox(container=container): + container_id: Final[object] = getattr(container, "id", None) + return container_id if isinstance(container_id, str) else None + case _SandboxUnavailable(): + return None @staticmethod def _agentic_tool_calls(tools: dict[str, object]) -> list[CodeExecutionToolCall]: @@ -588,14 +617,13 @@ class CodeInterpreterInterceptionLogger(CustomLogger): async def _build_chat_completion_tool_result( self, - container: ContainerHandle, - params: SandboxToolParams | None, + sandbox: _Sandbox, tool_call: CodeExecutionToolCall, container_id: str | None, ) -> tuple[ChatCompletionToolMessage, CodeInterpreterCall]: arguments: Final = tool_call.get("arguments", "") code: Final = self._parse_code(arguments) - stdout: Final = await self._run_tool_call(container=container, params=params, arguments=arguments) + stdout: Final = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments) tool_call_id: Final = tool_call.get("id") or tool_call.get("call_id") or uuid.uuid4().hex return ( { @@ -694,19 +722,36 @@ class CodeInterpreterInterceptionLogger(CustomLogger): except (json.JSONDecodeError, TypeError, AttributeError): return "" + async def _sandbox_tool_output(self, sandbox: _Sandbox, arguments: str) -> str: + match sandbox: + case _SandboxUnavailable(): + return _unavailable_output(sandbox) + case _ReadySandbox(container=container, params=params): + return await self._run_tool_call(container=container, params=params, arguments=arguments) + async def _run_tool_call(self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str) -> str: try: code: Final = json.loads(arguments).get("code", "") if arguments else "" except (json.JSONDecodeError, TypeError): return "[invalid tool arguments: could not parse code]" - result: Final = await self._run_code(container=container, params=params, code=code) + try: + result: Final = await self._run_code(container=container, params=params, code=code) + except Exception as e: + return _unavailable_output(_sandbox_unavailable("running the code", e)) if getattr(result, "error", None): error: Final = result.error message: Final = error.get("value") or error.get("name") if isinstance(error, dict) else str(error) return f"[execution error] {message}" return getattr(result, "stdout", "") or "" + async def _acquire_sandbox(self, cache_key: str | None, identity: str | None) -> _Sandbox: + try: + container, params = await self._get_or_create_container(cache_key=cache_key, identity=identity) + except Exception as e: + return _sandbox_unavailable("creating the sandbox", e) + return _ReadySandbox(container=container, params=params) + async def _get_or_create_container( self, cache_key: str | None, diff --git a/litellm/litellm_core_utils/agentic_loop_refusal.py b/litellm/litellm_core_utils/agentic_loop_refusal.py index 420a31fd516..ac3faad5f54 100644 --- a/litellm/litellm_core_utils/agentic_loop_refusal.py +++ b/litellm/litellm_core_utils/agentic_loop_refusal.py @@ -1,7 +1,10 @@ -from collections.abc import Iterable +import asyncio +from collections.abc import Awaitable, Iterable, Mapping from itertools import chain +from types import MappingProxyType from typing import Final +from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import Choices, ModelResponse @@ -11,9 +14,29 @@ def _strings(values: Iterable[object]) -> frozenset[str]: return frozenset(value for value in values if isinstance(value, str)) -def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]: +def _claimed_calls(tool_calls: object) -> tuple[object, ...]: wrapped: Final = message_field(tool_calls, "tool_calls") - calls: Final = parts_of(tool_calls if wrapped is None else wrapped) + return parts_of(tool_calls if wrapped is None else wrapped) + + +async def _calls_a_gate_claims(gate: Awaitable[tuple[bool, object]]) -> tuple[object, ...]: + try: + should_run, tool_calls = await gate + except Exception as e: + verbose_logger.debug("LiteLLM.AgenticLoopRefused: skipping a gate that failed: %s", e) + return () + return _claimed_calls(tool_calls) if should_run else () + + +async def every_claimed_tool_call( + refused_tool_calls: object, other_gates: Iterable[Awaitable[tuple[bool, object]]] +) -> Mapping[str, tuple[object, ...]]: + claims: Final = await asyncio.gather(*(_calls_a_gate_claims(gate) for gate in other_gates)) + return MappingProxyType({"tool_calls": (*_claimed_calls(refused_tool_calls), *chain.from_iterable(claims))}) + + +def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]: + calls: Final = _claimed_calls(tool_calls) ids: Final = _strings( chain.from_iterable((message_field(call, "id"), message_field(call, "call_id")) for call in calls) ) diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index b094010fc14..968d5d7293f 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -1,17 +1,17 @@ # this is a patch to allow for agentic loops covering llm_http_handler.py and openai sdk based calling flows for the .completion() api import json -from collections.abc import Mapping +from collections.abc import Awaitable, Iterator, Mapping, Sequence from itertools import chain from types import MappingProxyType -from typing import Final +from typing import Final, TypeVar from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs -from litellm.litellm_core_utils.agentic_loop_refusal import finalize_refused_chat_turn +from litellm.litellm_core_utils.agentic_loop_refusal import every_claimed_tool_call, finalize_refused_chat_turn from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, @@ -31,6 +31,8 @@ from litellm.types.integrations.custom_logger import ( from litellm.types.utils import ModelResponse from litellm.utils import CustomStreamWrapper +_DispatchResult = TypeVar("_DispatchResult") + _FOLLOWUP_INTERNAL_PARAMS: Final = frozenset( ( "acompletion", @@ -44,22 +46,36 @@ _FOLLOWUP_INTERNAL_PARAMS: Final = frozenset( ) +async def bill_first_round_on_failure( + dispatch: Awaitable[_DispatchResult], *, logging_obj: object, response: object, stream: bool +) -> _DispatchResult: + try: + return await dispatch + except Exception: + if not stream and isinstance(logging_obj, LiteLLMLoggingObject): + logging_obj.record_earlier_round_for_failure(response) + raise + + def _gate_overridden(callback: CustomLogger) -> bool: - base: Final = CustomLogger.async_should_run_agentic_loop - func: Final = type(callback).async_should_run_agentic_loop - return getattr(func, "__func__", func) is not getattr(base, "__func__", base) + return type(callback).async_should_run_agentic_loop is not CustomLogger.async_should_run_agentic_loop def _build_plan_overridden(callback: CustomLogger) -> bool: - base: Final = CustomLogger.async_build_agentic_loop_plan - func: Final = type(callback).async_build_agentic_loop_plan - return getattr(func, "__func__", func) is not getattr(base, "__func__", base) + return type(callback).async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan + + +def _chat_build_plan_overridden(callback: CustomLogger) -> bool: + return ( + type(callback).async_build_chat_completion_agentic_loop_plan + is not CustomLogger.async_build_chat_completion_agentic_loop_plan + ) def _post_hook_overridden(callback: CustomLogger) -> bool: - base: Final = CustomLogger.async_post_agentic_loop_response_hook - func: Final = type(callback).async_post_agentic_loop_response_hook - return getattr(func, "__func__", func) is not getattr(base, "__func__", base) + return ( + type(callback).async_post_agentic_loop_response_hook is not CustomLogger.async_post_agentic_loop_response_hook + ) _STREAM_OPTIONS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None) @@ -76,7 +92,7 @@ def _coerce_int(value: object, default: int) -> int: return int(value) if isinstance(value, (int, str)) else default -def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]: +def _agentic_loop_settings(kwargs: Mapping[str, object]) -> tuple[int, int, list[str]]: depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0) configured: Final = validated_max_agentic_loops( kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops" @@ -154,6 +170,59 @@ def _end_refused_turn( return finalized if replay is None else replay +def _custom_loggers(logging_obj: object) -> tuple[CustomLogger, ...]: + import litellm + + dynamic: Final = getattr(logging_obj, "dynamic_success_callbacks", None) or [] + return tuple(callback for callback in (*litellm.callbacks, *dynamic) if isinstance(callback, CustomLogger)) + + +async def _refused_chat_turn( + callbacks: Sequence[CustomLogger], + refusing: CustomLogger, + refused_tool_calls: object, + *, + response: ModelResponse, + model: str, + messages: list, + tools: list[dict[str, object]] | None, + custom_llm_provider: str, + kwargs: Mapping[str, object], +) -> ModelResponse: + surface_kwargs: Final = { + **kwargs, + "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, + "custom_llm_provider": custom_llm_provider, + } + + def other_gates() -> Iterator[Awaitable[tuple[bool, object]]]: + for callback in callbacks: + if callback is refusing: + continue + yield callback.async_should_run_chat_completion_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=False, + custom_llm_provider=custom_llm_provider, + kwargs=dict(kwargs), + ) + if _gate_overridden(callback): + yield callback.async_should_run_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=False, + custom_llm_provider=custom_llm_provider, + kwargs=dict(surface_kwargs), + ) + + claimed: Final = await every_claimed_tool_call(refused_tool_calls, other_gates()) + return finalize_refused_chat_turn(response, claimed) + + def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]: metadata: Final = kwargs_for_followup.get("litellm_metadata") return MappingProxyType( @@ -280,16 +349,11 @@ async def maybe_run_chat_completion_agentic_loop( custom_llm_provider: str, stream: bool, ) -> ModelResponse | CustomStreamWrapper | None: - import litellm - - callbacks: Final = litellm.callbacks + (getattr(logging_obj, "dynamic_success_callbacks", None) or []) + callbacks: Final = _custom_loggers(logging_obj) depth, max_loops, fingerprints = _agentic_loop_settings(kwargs) tools: Final = optional_params.get("tools", []) for callback in callbacks: - if not isinstance(callback, CustomLogger): - continue - if not _gate_overridden(callback): continue @@ -331,7 +395,17 @@ async def maybe_run_chat_completion_agentic_loop( raise verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [model=%s]: %s", model, str(e)) return _end_refused_turn( - finalize_refused_chat_turn(response, tool_calls), + await _refused_chat_turn( + callbacks, + callback, + tool_calls, + response=response, + model=model, + messages=messages, + tools=tools, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + ), kwargs=kwargs, depth=depth, model=model, @@ -393,3 +467,113 @@ async def maybe_run_chat_completion_agentic_loop( custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, ) + + +async def run_chat_completion_agentic_hooks( + *, + response: ModelResponse, + model: str, + messages: list, + optional_params: dict, + kwargs: dict[str, object], + logging_obj: LiteLLMLoggingObject, + custom_llm_provider: str, + stream: bool, +) -> object | None: + callbacks: Final = _custom_loggers(logging_obj) + depth, max_loops, fingerprints = _agentic_loop_settings(kwargs) + tools: Final = optional_params.get("tools", []) + hook_kwargs: Final = {**kwargs, "custom_llm_provider": custom_llm_provider} + + for callback in callbacks: + try: + should_run, tool_calls = await callback.async_should_run_chat_completion_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=stream, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + ) + except Exception as e: + verbose_logger.exception( + "LiteLLM.AgenticHookError: Exception in async_should_run_chat_completion_agentic_loop: %s", str(e) + ) + continue + + if not should_run: + continue + + try: + fingerprint = _check_agentic_loop_safety( + tool_calls=tool_calls, + fingerprints=fingerprints, + depth=depth, + max_loops=max_loops, + model=model, + ) + except AgenticLoopSafetyError as e: + if stream: + raise + verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [model=%s]: %s", model, str(e)) + return await _refused_chat_turn( + callbacks, + callback, + tool_calls, + response=response, + model=model, + messages=messages, + tools=tools, + custom_llm_provider=custom_llm_provider, + kwargs=kwargs, + ) + + if not _chat_build_plan_overridden(callback): + return await callback.async_run_chat_completion_agentic_loop( + tools=tool_calls, + model=model, + messages=messages, + response=response, + optional_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs=hook_kwargs, + ) + + plan = await callback.async_build_chat_completion_agentic_loop_plan( + tools=tool_calls, + model=model, + messages=messages, + response=response, + optional_params=optional_params, + logging_obj=logging_obj, + stream=stream, + kwargs=hook_kwargs, + ) + + if plan.response_override is not None: + return plan.response_override + if plan.terminate: + verbose_logger.debug( + "Agentic chat loop terminated by callback=%s reason=%s", type(callback).__name__, plan.stop_reason + ) + return response + if not plan.run_agentic_loop: + continue + + return await _execute_chat_completion_agentic_plan( + plan=plan, + callback=callback, + model=model, + optional_params=optional_params, + kwargs=hook_kwargs, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + ) + + return None diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 8178b7e98b8..e0744ceaa71 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -33,6 +33,9 @@ AWS_CREDENTIAL_KWARGS_KEYS: Final = frozenset( ) PROVIDER_AFFINITY_HEADER_KWARG_KEY: Final = "provider_affinity_header" +AGENTIC_LOOP_CONTROL_KWARGS_KEYS: Final = frozenset( + {"_agentic_loop_depth", "_agentic_loop_fingerprints", "max_agentic_loops"} +) OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS: Final = frozenset( { "token_exchange_endpoint", @@ -91,6 +94,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( | ANTHROPIC_WIF_KWARGS_KEYS | OPENAI_WIF_KWARGS_KEYS | OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS + | AGENTIC_LOOP_CONTROL_KWARGS_KEYS | frozenset(CustomPricingLiteLLMParams.model_fields) ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 620c8413f29..dc0fe85aa2c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -2565,6 +2565,11 @@ class Logging(LiteLLMLoggingBaseClass): self._stream_failure_claimed = True return True + def record_earlier_round_for_failure(self, response: object) -> None: + usage: Final = get_usage_object(completion_response=response) + if usage is not None: + self.record_partial_usage_for_failure(usage, self.response_cost_calculator(result=response) or 0.0) + def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None: """Bill a fully streamed response on the failure log when a post-call hook rejects it.""" usage: Final = getattr(assembled, "usage", None) diff --git a/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py b/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py index 25d4d7b3ce1..c33a849b5b3 100644 --- a/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py @@ -22,6 +22,7 @@ from typing import TYPE_CHECKING, Any, Final, cast from litellm._logging import verbose_logger from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES +from litellm.types.integrations.custom_logger import AgenticLoopSafetyError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -393,11 +394,10 @@ class AgenticAnthropicStreamingIterator: "AgenticStreamingIterator: Unexpected result type from hooks: %s", type(result).__name__, ) - except Exception as e: - _call_id: Final = getattr(self._logging_obj, "litellm_call_id", "unknown") - verbose_logger.exception( - "AgenticStreamingIterator: Error in agentic hook processing [call_id=%s model=%s]: %s", - _call_id, + except AgenticLoopSafetyError as e: + verbose_logger.warning( + "AgenticStreamingIterator: agentic loop refused [call_id=%s model=%s]: %s", + getattr(self._logging_obj, "litellm_call_id", "unknown"), self._model, str(e), ) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 4f7792ec4d6..e0cac4e1059 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -42,6 +42,7 @@ from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAG from litellm.files.types import FileContentStreamingResult from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs from litellm.litellm_core_utils.agentic_loop_refusal import ( + every_claimed_tool_call, finalize_refused_responses_turn, is_refused_item, refused_tool_call_identifiers, @@ -55,6 +56,10 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SUBTITLE_RESPONSE_FORMATS, synthesize_subtitle_document, ) +from litellm.litellm_core_utils.chat_completion_agentic_loop import ( + bill_first_round_on_failure, + run_chat_completion_agentic_hooks, +) from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS @@ -6143,66 +6148,6 @@ class BaseLLMHTTPHandler: custom_llm_provider=custom_llm_provider, ) - async def _execute_chat_completion_agentic_plan( - self, - plan: AgenticLoopPlan, - model: str, - messages: list[dict], - optional_params: dict, - kwargs: dict, - custom_llm_provider: str, - depth: int, - max_loops: int, - fingerprints: list[str], - fingerprint: str, - ) -> ModelResponse | CustomStreamWrapper: - patch: Final = plan.request_patch or AgenticLoopRequestPatch() - if patch.messages is None: - raise ValueError("Agentic loop plan missing patched messages") - - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" - - optional_params_for_followup: Final = dict(optional_params) - optional_params_for_followup.update(patch.optional_params) - if patch.tools is not None: - optional_params_for_followup["tools"] = patch.tools - - internal_params: Final = { - "_websearch_interception", - "acompletion", - "litellm_logging_obj", - "custom_llm_provider", - "model_alias_map", - "stream_response", - "custom_prompt_dict", - } - kwargs_for_followup: Final = build_agentic_followup_kwargs( - request_kwargs=MappingProxyType( - { - k: v - for k, v in kwargs.items() - if not k.startswith("_websearch_interception") - and not k.startswith("_compression_interception") - and k not in internal_params - } - ), - patch_kwargs=patch.kwargs, - request_params=frozenset((*optional_params_for_followup, "model", "messages")), - depth=depth, - max_loops=max_loops, - fingerprints=fingerprints, - fingerprint=fingerprint, - ) - - return await litellm.acompletion( - model=full_model_name, - messages=patch.messages, - **optional_params_for_followup, - **kwargs_for_followup, - ) - def _maybe_wrap_in_fake_stream( self, response: _ResponseT, @@ -6326,7 +6271,25 @@ class BaseLLMHTTPHandler: str(e), ) return self._maybe_wrap_in_fake_stream( - self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls), + self._finalize_refused_agentic_response( + response=response, + tool_calls=await every_claimed_tool_call( + tool_calls, + ( + other.async_should_run_agentic_loop( + response=response, + model=model, + messages=messages, + tools=tools, + stream=stream, + custom_llm_provider=custom_llm_provider, + kwargs=hook_kwargs, + ) + for other in callbacks + if isinstance(other, CustomLogger) and other is not callback + ), + ), + ), logging_obj, api_surface, ) @@ -6336,16 +6299,21 @@ class BaseLLMHTTPHandler: callback.__class__.async_build_agentic_loop_plan is not CustomLogger.async_build_agentic_loop_plan ) if not build_plan_overridden: - agentic_result: object = await callback.async_run_agentic_loop( - tools=tool_calls, - model=model, - messages=messages, - response=response, - anthropic_messages_provider_config=anthropic_messages_provider_config, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + agentic_result: object = await bill_first_round_on_failure( + callback.async_run_agentic_loop( + tools=tool_calls, + model=model, + messages=messages, + response=response, + anthropic_messages_provider_config=anthropic_messages_provider_config, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + logging_obj=logging_obj, + stream=stream, + kwargs=kwargs_with_provider, + ), logging_obj=logging_obj, + response=response, stream=stream, - kwargs=kwargs_with_provider, ) return self._maybe_wrap_in_fake_stream(agentic_result, logging_obj, api_surface) @@ -6382,33 +6350,43 @@ class BaseLLMHTTPHandler: continue if api_surface == "responses": - return await self._execute_responses_agentic_plan( - plan=plan, - model=model, - response_api_optional_request_params=anthropic_messages_optional_request_params, + return await bill_first_round_on_failure( + self._execute_responses_agentic_plan( + plan=plan, + model=model, + response_api_optional_request_params=anthropic_messages_optional_request_params, + logging_obj=logging_obj, + kwargs=kwargs_with_provider, + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + callback=callback, + ), logging_obj=logging_obj, - kwargs=kwargs_with_provider, - depth=depth, - max_loops=max_loops, - fingerprints=fingerprints, - fingerprint=fingerprint, - callback=callback, + response=response, + stream=stream, ) return self._maybe_wrap_in_fake_stream( - await self._execute_anthropic_agentic_plan( - plan=plan, - model=model, - messages=messages, - anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + await bill_first_round_on_failure( + self._execute_anthropic_agentic_plan( + plan=plan, + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + logging_obj=logging_obj, + kwargs=kwargs_with_provider, + depth=depth, + max_loops=max_loops, + fingerprints=fingerprints, + fingerprint=fingerprint, + stream=stream, + callback=callback, + ), logging_obj=logging_obj, - kwargs=kwargs_with_provider, - depth=depth, - max_loops=max_loops, - fingerprints=fingerprints, - fingerprint=fingerprint, + response=response, stream=stream, - callback=callback, ), logging_obj, api_surface, @@ -6444,106 +6422,23 @@ class BaseLLMHTTPHandler: Returns the response from agentic loop, or None if no hook runs. """ - from litellm._logging import verbose_logger - from litellm.integrations.custom_logger import CustomLogger - - callbacks: Final = litellm.callbacks + (logging_obj.dynamic_success_callbacks or []) - tools: Final = optional_params.get("tools", []) - depth, max_loops, fingerprints = self._get_agentic_loop_settings(kwargs=kwargs) - - for callback in callbacks: - if not isinstance(callback, CustomLogger): - continue - if not hasattr(callback, "async_should_run_chat_completion_agentic_loop"): - continue - - should_run: bool = False - tool_calls: object = None - try: - ( - should_run, - tool_calls, - ) = await callback.async_should_run_chat_completion_agentic_loop( - response=response, - model=model, - messages=messages, - tools=tools, - stream=stream, - custom_llm_provider=custom_llm_provider, - kwargs=kwargs, - ) - except Exception as e: - verbose_logger.exception( - "LiteLLM.AgenticHookError: Exception in async_should_run_chat_completion_agentic_loop: %s", - str(e), - ) - continue - - if not should_run: - continue - - # Safety guards must run OUTSIDE the callback try/except — they are - # bounded-loop / cycle-break rails that must propagate to the caller. - fingerprint = self._check_agentic_loop_safety( - tool_calls=tool_calls, - fingerprints=fingerprints, - depth=depth, - max_loops=max_loops, - model=model, - ) - - kwargs_with_provider = {**(kwargs or {}), "custom_llm_provider": custom_llm_provider} - build_plan_overridden = ( - callback.__class__.async_build_chat_completion_agentic_loop_plan - is not CustomLogger.async_build_chat_completion_agentic_loop_plan - ) - if not build_plan_overridden: - return await callback.async_run_chat_completion_agentic_loop( - tools=tool_calls, - model=model, - messages=messages, - response=response, - optional_params=optional_params, - logging_obj=logging_obj, - stream=stream, - kwargs=kwargs_with_provider, - ) - - plan = await callback.async_build_chat_completion_agentic_loop_plan( - tools=tool_calls, - model=model, - messages=messages, + agentic_response: Final = await bill_first_round_on_failure( + run_chat_completion_agentic_hooks( response=response, - optional_params=optional_params, - logging_obj=logging_obj, - stream=stream, - kwargs=kwargs_with_provider, - ) - - if plan.response_override is not None: - return plan.response_override - if plan.terminate: - verbose_logger.debug( - "Agentic chat loop terminated by callback=%s reason=%s", - callback.__class__.__name__, - plan.stop_reason, - ) - return response - if not plan.run_agentic_loop: - continue - - return await self._execute_chat_completion_agentic_plan( - plan=plan, model=model, messages=messages, optional_params=optional_params, - kwargs=kwargs_with_provider, + kwargs=kwargs or {}, + logging_obj=logging_obj, custom_llm_provider=custom_llm_provider, - depth=depth, - max_loops=max_loops, - fingerprints=fingerprints, - fingerprint=fingerprint, - ) + stream=stream, + ), + logging_obj=logging_obj, + response=response, + stream=stream, + ) + if agentic_response is not None: + return agentic_response # Check if we need to convert response to fake stream for chat completions # This happens when: diff --git a/litellm/llms/openai/openai.py b/litellm/llms/openai/openai.py index d36ae19b698..3c6a2fd94b2 100644 --- a/litellm/llms/openai/openai.py +++ b/litellm/llms/openai/openai.py @@ -28,6 +28,10 @@ from litellm import LlmProviders from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RETRIES from litellm.files.types import FileContentStreamingResult +from litellm.litellm_core_utils.chat_completion_agentic_loop import ( + bill_first_round_on_failure, + run_chat_completion_agentic_hooks, +) from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing @@ -631,52 +635,23 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): Returns the response from agentic loop, or None if no hook runs. """ - from litellm._logging import verbose_logger - from litellm.integrations.custom_logger import CustomLogger - - callbacks: Final = litellm.callbacks + (logging_obj.dynamic_success_callbacks or []) - # Avoid logging full callback objects to prevent leaking sensitive data - verbose_logger.debug("LiteLLM.AgenticHooks: callbacks_count=%s", len(callbacks)) - tools: Final = optional_params.get("tools", []) - # Avoid logging full tools payloads; they may contain sensitive parameters - verbose_logger.debug( - "LiteLLM.AgenticHooks: tools_count=%s", - len(tools) if isinstance(tools, list) else 1 if tools else 0, + if not isinstance(response, ModelResponse): + return None + return await bill_first_round_on_failure( + run_chat_completion_agentic_hooks( + response=response, + model=model, + messages=messages, + optional_params=optional_params, + kwargs=litellm_params or {}, + logging_obj=logging_obj, + custom_llm_provider=litellm_params.get("custom_llm_provider", "openai"), + stream=stream, + ), + logging_obj=logging_obj, + response=response, + stream=stream, ) - # Get custom_llm_provider from litellm_params - custom_llm_provider: Final = litellm_params.get("custom_llm_provider", "openai") - - custom_loggers: Final = tuple(callback for callback in callbacks if isinstance(callback, CustomLogger)) - for callback in custom_loggers: - try: - should_run, tool_calls = await callback.async_should_run_chat_completion_agentic_loop( - response=response, - model=model, - messages=messages, - tools=tools, - stream=stream, - custom_llm_provider=custom_llm_provider, - kwargs=litellm_params, - ) - except Exception as e: - verbose_logger.exception( - "LiteLLM.AgenticHookError: Exception in OpenAI chat completion agentic gate: %s", e - ) - continue - - if should_run: - return await callback.async_run_chat_completion_agentic_loop( - tools=tool_calls, - model=model, - messages=messages, - response=response, - optional_params=optional_params, - logging_obj=logging_obj, - stream=stream, - kwargs={**(litellm_params or {}), "custom_llm_provider": custom_llm_provider}, - ) - - return None def mock_streaming( self, @@ -1025,12 +1000,10 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM): _response_headers=headers, ) except openai.UnprocessableEntityError as e: - ## check if body contains unprocessable params - related issue https://github.com/BerriAI/litellm/issues/4800 if litellm.drop_params is True or drop_params is True: data = drop_params_from_unprocessable_entity_error(e, data) else: raise e - # e.message except Exception as e: exception_response: object = getattr(e, "response", None) status_code = getattr(e, "status_code", 500) diff --git a/litellm/main.py b/litellm/main.py index 41c4302499e..ffd5d4d797e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -75,11 +75,13 @@ from litellm.litellm_core_utils.audio_utils.utils import ( get_audio_file_for_health_check, ) from litellm.litellm_core_utils.chat_completion_agentic_loop import ( + bill_first_round_on_failure, maybe_run_chat_completion_agentic_loop, ) from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_litellm_params import ( + AGENTIC_LOOP_CONTROL_KWARGS_KEYS, AWS_CREDENTIAL_KWARGS_KEYS, OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, OPTIONAL_KWARGS_KEYS, @@ -758,26 +760,31 @@ async def acompletion( # no equivalent: every provider already funnels through one shared # handler where the loop is dispatched. if isinstance(response, litellm.ModelResponse): - looped: Final = await maybe_run_chat_completion_agentic_loop( - response=response, - model=model, - messages=messages, - optional_params={ - k: v - for k, v in completion_kwargs.items() - if v is not None - and k - not in ( - "model", - "messages", - "stream", - "acompletion", - "deployment_id", - ) - }, - kwargs=kwargs, + looped: Final = await bill_first_round_on_failure( + maybe_run_chat_completion_agentic_loop( + response=response, + model=model, + messages=messages, + optional_params={ + k: v + for k, v in completion_kwargs.items() + if v is not None + and k + not in ( + "model", + "messages", + "stream", + "acompletion", + "deployment_id", + ) + }, + kwargs=kwargs, + logging_obj=kwargs.get("litellm_logging_obj"), + custom_llm_provider=custom_llm_provider, + stream=bool(stream), + ), logging_obj=kwargs.get("litellm_logging_obj"), - custom_llm_provider=custom_llm_provider, + response=response, stream=bool(stream), ) if looped is not None: @@ -6028,6 +6035,7 @@ def completion( *ANTHROPIC_WIF_KWARGS_KEYS, *OPENAI_WIF_KWARGS_KEYS, *OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS, + *AGENTIC_LOOP_CONTROL_KWARGS_KEYS, "github_copilot_auth_type", "github_copilot_host", "github_copilot_user_session", diff --git a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py index 4d910cde603..9b9e0949a1a 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py +++ b/litellm/proxy/guardrails/guardrail_hooks/compresr/compresr.py @@ -22,7 +22,7 @@ import json import time from collections import Counter, OrderedDict from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeGuard +from typing import TYPE_CHECKING, ClassVar, Final, Literal, Protocol, TypeGuard from urllib.parse import urlparse import httpx @@ -507,6 +507,8 @@ class _CompressionResult: class CompresrGuardrail(CustomGuardrail): + server_fulfilled_tool_names: ClassVar[frozenset[str]] = frozenset({COMPRESR_RETRIEVE_TOOL_NAME}) + def __init__( self, api_base: str | None = None, diff --git a/tests/integration/streaming/test_interception_converted_stream_options.py b/tests/integration/streaming/test_interception_converted_stream_options.py index 85b65683ba6..7b0014f8073 100644 --- a/tests/integration/streaming/test_interception_converted_stream_options.py +++ b/tests/integration/streaming/test_interception_converted_stream_options.py @@ -216,10 +216,17 @@ def scenario(rig: _Rig) -> Iterator[Scenario]: yield opened -def _deploy(rig: _Rig, scenario: Scenario, mode: iv.Mode, *, api_key: str = iv.AZURE_KEY) -> _Deployment: +def _deploy( + rig: _Rig, + scenario: Scenario, + mode: iv.Mode, + *, + api_key: str = iv.AZURE_KEY, + pricing: Mapping[str, JsonValue] = MappingProxyType({}), +) -> _Deployment: name: Final = iv.deployment(mode) model: Final = scenario.model( - model=f"azure/{name}", api_base=rig.wire.url, api_key=api_key, api_version=iv.API_VERSION + model=f"azure/{name}", api_base=rig.wire.url, api_key=api_key, api_version=iv.API_VERSION, **pricing ) return _Deployment(model, name) @@ -754,27 +761,33 @@ def test_wrong_azure_key_surfaces_azures_401(rig: _Rig, scenario: Scenario) -> N assert len(_upstream(rig, deployment).model_calls) == 1 -def _assert_failed_once(call_id: str) -> None: +def _assert_failed_once(call_id: str, *, spend: float) -> None: rows: Final = eventually( - lambda: read_rows('SELECT status FROM "LiteLLM_SpendLogs" WHERE litellm_call_id = %s', (call_id,)), + lambda: read_rows('SELECT status, spend FROM "LiteLLM_SpendLogs" WHERE litellm_call_id = %s', (call_id,)), lambda found: len(found) >= 1, seconds=70, ) assert [row["status"] for row in rows] == ["failure"], rows + assert float(rows[0]["spend"]) == pytest.approx(spend), rows @pytest.mark.parametrize("stream", [True, False], ids=["stream", "json"]) def test_failed_follow_up_call_answers_with_the_providers_error_instead_of_the_search_tool_call( rig: _Rig, scenario: Scenario, stream: bool ) -> None: - deployment: Final = _deploy(rig, scenario, "followup-fails") + deployment: Final = _deploy( + rig, scenario, "followup-fails", pricing={"input_cost_per_token": 0.001, "output_cost_per_token": 0.002} + ) extra: Final = {**_NO_CACHE, "stream_options": dict(_USAGE)} if stream else _NO_CACHE body: Final = _chat_body(deployment, stream=stream, extra=extra) response: Final = rig.gateway.request("POST", "/v1/chat/completions", body) assert response.status_code == 500 and iv.FOLLOWUP_FAILURE in response.text, response.text assert iv.SEARCH_TOOL not in response.text, response.text _assert_upstream(_upstream(rig, deployment), model_calls=2, searches=1) - _assert_failed_once(response.headers["x-litellm-call-id"]) + _assert_failed_once( + response.headers["x-litellm-call-id"], + spend=iv.FIRST_USAGE["prompt_tokens"] * 0.001 + iv.FIRST_USAGE["completion_tokens"] * 0.002, + ) assert rig.gateway.chat(_deploy(rig, scenario, "plain").model)["object"] == "chat.completion" diff --git a/tests/unit/integrations/code_interpreter_interception/test_handler.py b/tests/unit/integrations/code_interpreter_interception/test_handler.py index c037b6be80d..6f94e845e69 100644 --- a/tests/unit/integrations/code_interpreter_interception/test_handler.py +++ b/tests/unit/integrations/code_interpreter_interception/test_handler.py @@ -831,11 +831,7 @@ async def test_post_hook_delete_is_idempotent_across_loop_levels(): @pytest.mark.asyncio -async def test_build_plan_deletes_sandbox_when_execution_raises(): - """If sandbox execution raises before a plan is built (e.g. E2B aborts - output over its cap), the cached sandbox must be deleted before re-raising, - otherwise a caller can leak paid containers until the prune TTL.""" - +async def test_a_failed_code_run_still_deletes_its_sandbox_once_the_loop_ends(): class RaisingSandbox(FakeSandbox): async def arun_code(self, *, container, code, **kwargs): raise ValueError("output exceeded cap") @@ -843,13 +839,11 @@ async def test_build_plan_deletes_sandbox_when_execution_raises(): sandbox = RaisingSandbox() logger = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) - with pytest.raises(ValueError, match="exceeded cap"): - await _build_plan(logger, sandbox, call_id="k1") + plan = await _build_plan(logger, sandbox, call_id="k1") + await logger.async_agentic_loop_cleanup_hook(plan=plan, kwargs={}) - assert len(sandbox.create_calls) == 1, "the sandbox must have been created" - assert len(sandbox.delete_calls) == 1, ( - "a build failure must delete the cached sandbox so it does not keep running and billing" - ) + assert len(sandbox.create_calls) == 1 + assert len(sandbox.delete_calls) == 1 assert "sbxkey1" not in logger._container_cache @@ -1448,3 +1442,63 @@ async def test_per_identity_cap_evicts_lru_session(): "the entry with the oldest last_accessed timestamp must be evicted first (LRU)" ) assert len(sandbox.delete_calls) == 1, "evicted sandbox must be deleted, not just removed from cache" + + +class _SandboxThatCannotStart(FakeSandbox): + async def acreate_sandbox(self, **kwargs): + raise ConnectionError("sandbox provider unreachable") + + +class _SandboxThatCannotRunCode(FakeSandbox): + async def arun_code(self, *, container, code, **kwargs): + raise TimeoutError("sandbox run timed out") + + +_SANDBOX_FAILURES: Final = ( + pytest.param(None, "[sandbox unavailable] CodeInterpreterInterception: no sandbox available", id="no_sandbox"), + pytest.param(_SandboxThatCannotStart(), "[sandbox unavailable] sandbox provider unreachable", id="create_fails"), + pytest.param(_SandboxThatCannotRunCode(), "[sandbox unavailable] sandbox run timed out", id="run_fails"), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("sandbox", "expected_output"), _SANDBOX_FAILURES) +async def test_a_sandbox_failure_answers_the_responses_tool_call_with_the_failure(sandbox, expected_output): + plan: Final = await _build_plan(CodeInterpreterInterceptionLogger(sandbox_config=sandbox), sandbox) + + outputs: Final = [message for message in _iter_messages(plan) if message.get("type") == "function_call_output"] + assert plan.run_agentic_loop is True + assert [(output["call_id"], output["output"].startswith(expected_output)) for output in outputs] == [("c1", True)] + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("sandbox", "expected_output"), _SANDBOX_FAILURES) +async def test_a_sandbox_failure_answers_the_chat_tool_call_with_the_failure(sandbox, expected_output): + logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) + + plan: Final = await logger.async_build_agentic_loop_plan( + tools={ + "tool_calls": [ + {"id": "call_1", "name": LITELLM_CODE_EXECUTION_TOOL_NAME, "arguments": '{"code":"print(40 + 2)"}'} + ] + }, + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + response={"choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]}, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={"tools": []}, + logging_obj=FakeLogging(litellm_call_id="k1"), + stream=False, + kwargs={ + "litellm_call_id": "k1", + _ACTIVE_KEY: True, + _SANDBOX_KEY: "sbxkey1", + "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE, + }, + ) + + tool_messages: Final = [message for message in _iter_messages(plan) if message.get("role") == "tool"] + assert plan.run_agentic_loop is True + assert [(message["tool_call_id"], message["content"].startswith(expected_output)) for message in tool_messages] == [ + ("call_1", True) + ] diff --git a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py index 60d77fd6dda..a100cb30fc0 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_agentic_loop_cap.py @@ -12,6 +12,7 @@ config.yaml through to the settings the loop actually reads. """ import json +import time from unittest.mock import MagicMock, patch import pytest @@ -24,6 +25,7 @@ from litellm.integrations.websearch_interception.handler import ( WebSearchInterceptionLogger, ) from litellm.integrations.websearch_interception.tools import get_litellm_web_search_tool +from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.agentic_loop_settings import DEFAULT_MAX_AGENTIC_LOOPS from litellm.llms.anthropic.messages.native.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, @@ -39,6 +41,7 @@ from litellm.types.integrations.custom_logger import ( ) INTERNAL_TOOL_NAME = "litellm_web_search" +CODE_EXECUTION_TOOL_NAME = "litellm_code_execution" @pytest.fixture(autouse=True) @@ -188,14 +191,34 @@ class _ResponsesInterceptingCallback(_InterceptingCallback): return bool(calls), {"tool_calls": calls} -def _logging_obj(callback: CustomLogger, converted_stream: bool = False) -> MagicMock: +def _logging_obj(*callbacks: CustomLogger, converted_stream: bool = False) -> MagicMock: logging_obj = MagicMock() logging_obj.model_call_details = {"websearch_interception_converted_stream": converted_stream} - logging_obj.dynamic_success_callbacks = [callback] + logging_obj.dynamic_success_callbacks = list(callbacks) logging_obj.litellm_call_id = "call-abc" return logging_obj +class _CodeExecutionGate(CustomLogger): + async def async_should_run_agentic_loop( + self, response, model, messages, tools, stream, custom_llm_provider, kwargs + ): + calls = ( + [ + {"id": block["id"], "name": block["name"]} + for block in response["content"] + if block.get("type") == "tool_use" and block.get("name") == CODE_EXECUTION_TOOL_NAME + ] + if isinstance(response, dict) + else [ + {"call_id": item.call_id, "name": item.name} + for item in response.output + if item.type == "function_call" and item.name == CODE_EXECUTION_TOOL_NAME + ] + ) + return bool(calls), {"tool_calls": calls} + + async def _run_hooks( handler: BaseLLMHTTPHandler, callback: CustomLogger, @@ -350,6 +373,48 @@ class TestCappedLoopReturnsTerminalResponse: assert _tool_use_names(result) == ["get_weather"] assert result["stop_reason"] == "tool_use" + @pytest.mark.asyncio + async def test_every_hooks_internal_call_is_dropped_and_the_clients_survives(self): + response = _response_asking_for_another_search() + response["content"] += [ + {"id": "toolu_code_1", "type": "tool_use", "name": CODE_EXECUTION_TOOL_NAME, "input": {}}, + {"id": "toolu_client_1", "type": "tool_use", "name": "get_weather", "input": {}}, + ] + + result = await self.handler._call_agentic_completion_hooks( + response=response, + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={}, + logging_obj=_logging_obj(self.callback, _CodeExecutionGate()), + stream=False, + custom_llm_provider="anthropic", + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + ) + + assert _tool_use_names(result) == ["get_weather"] + assert self.callback.plan_calls == 0 + + @pytest.mark.asyncio + async def test_responses_surface_drops_every_hooks_internal_call_and_keeps_the_clients(self): + result = await self.handler._call_agentic_completion_hooks( + response=_responses_turn( + ("call_search", INTERNAL_TOOL_NAME), ("call_code", CODE_EXECUTION_TOOL_NAME), ("call_client", "lookup") + ), + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={}, + logging_obj=_logging_obj(_ResponsesInterceptingCallback(), _CodeExecutionGate()), + stream=False, + custom_llm_provider="anthropic", + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + api_surface="responses", + ) + + assert [getattr(item, "call_id", None) for item in result.output] == [None, "call_client"] + def test_only_the_refused_tool_calls_are_dropped(self): """ A block is matched on the id the rail refused, not on the tool name, so a @@ -384,11 +449,6 @@ class TestCappedLoopReturnsTerminalResponse: assert result["stop_reason"] == "end_turn" def test_a_name_only_refusal_keeps_the_native_search_blocks_sharing_that_name(self): - """ - The gate also picks up a bare web_search tool_use, and the native - server_tool_use blocks already in the turn carry that same name. Only the - tool_use goes, so each web_search_tool_result keeps the block it answers. - """ response = _response_asking_for_another_search() response["content"][-1] = {"type": "tool_use", "name": "web_search", "input": {}} @@ -438,11 +498,6 @@ class TestCappedLoopReturnsTerminalResponse: assert callback.plan_calls == 0 def test_responses_refusal_by_call_id_alone_keeps_the_calls_it_did_not_list(self): - """ - The code interpreter gate hands over call_id and name with no id, so the - call_id is what scopes the refusal there, the same way the id does on - /v1/messages. - """ response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_unlisted", INTERNAL_TOOL_NAME)) result = BaseLLMHTTPHandler._finalize_refused_agentic_response( @@ -582,13 +637,32 @@ def _response_asking_for_searches(*queries: str) -> dict: } -class TestFailedFollowUpCallReachesTheClient: - """ - A follow-up model call that fails after a successful search used to be - logged and dropped, so the client got the pre-loop turn holding a - litellm_web_search tool_use it never declared. - """ +def _request_logging_obj(call_type: str, *callbacks: CustomLogger) -> Logging: + logging_obj = Logging( + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + stream=False, + call_type=call_type, + start_time=time.time(), + litellm_call_id="call-abc", + function_id="f", + dynamic_success_callbacks=list(callbacks), + ) + logging_obj.update_environment_variables( + model="claude-sonnet-4-5", user="", optional_params={}, litellm_params={}, custom_llm_provider="anthropic" + ) + return logging_obj + +def _assert_first_call_billed(logging_obj: Logging, *, input_tokens: int, output_tokens: int) -> None: + price = litellm.model_cost["claude-sonnet-4-5"] + assert logging_obj.model_call_details["combined_usage_object"].total_tokens == input_tokens + output_tokens + assert logging_obj.model_call_details["response_cost"] == pytest.approx( + input_tokens * price["input_cost_per_token"] + output_tokens * price["output_cost_per_token"] + ) + + +class TestFailedFollowUpCallReachesTheClient: def setup_method(self): self.handler = BaseLLMHTTPHandler() self.logger = WebSearchInterceptionLogger(enabled_providers=["anthropic"]) @@ -625,6 +699,54 @@ class TestFailedFollowUpCallReachesTheClient: kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True}, ) + @pytest.mark.asyncio + async def test_follow_up_rate_limit_bills_the_first_call_on_the_failure_log(self, monkeypatch): + monkeypatch.setattr("litellm.anthropic_interface.messages.acreate", self._rate_limited_follow_up) + logging_obj = _request_logging_obj("anthropic_messages", self.logger) + + with ( + patch.object(self.logger, "_execute_search", side_effect=self._search), + pytest.raises(RateLimitError), + ): + await self.handler._call_agentic_completion_hooks( + response=_response_asking_for_searches("who won the world cup"), + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={"tools": [get_litellm_web_search_tool()]}, + logging_obj=logging_obj, + stream=False, + custom_llm_provider="anthropic", + kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, WEBSEARCH_EMIT_NATIVE_BLOCKS_KEY: True}, + ) + + _assert_first_call_billed(logging_obj, input_tokens=10, output_tokens=5) + + @pytest.mark.asyncio + async def test_responses_follow_up_rate_limit_bills_the_first_call_on_the_failure_log(self, monkeypatch): + monkeypatch.setattr(litellm, "aresponses", self._rate_limited_follow_up) + callback = _ResponsesInterceptingCallback() + logging_obj = _request_logging_obj("aresponses", callback) + first_call = _responses_turn(("call_search", INTERNAL_TOOL_NAME)).model_copy( + update={"usage": {"input_tokens": 13, "output_tokens": 6, "total_tokens": 19}} + ) + + with pytest.raises(RateLimitError): + await self.handler._call_agentic_completion_hooks( + response=first_call, + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={}, + logging_obj=logging_obj, + stream=False, + custom_llm_provider="anthropic", + kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3}, + api_surface="responses", + ) + + _assert_first_call_billed(logging_obj, input_tokens=13, output_tokens=6) + @pytest.mark.asyncio async def test_responses_follow_up_rate_limit_is_raised_to_the_caller(self, monkeypatch): monkeypatch.setattr(litellm, "aresponses", self._rate_limited_follow_up) diff --git a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py index acb3987341f..2b475bc9922 100644 --- a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -52,6 +52,7 @@ from litellm.types.utils import ( Function, Message, ModelResponse, + Usage, ) # The internal control fields that must never reach a provider request body. @@ -420,8 +421,12 @@ async def _run_refused_rerun( rail_kwargs: Dict[str, object], stream: bool = False, gate_tool_calls: Optional[Dict[str, object]] = _REFUSED_GATE_TOOL_CALLS, + other_hooks_tool_calls: tuple[Dict[str, object], ...] = (), ) -> Any: - litellm.callbacks = [_GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=True), tool_calls=gate_tool_calls)] + litellm.callbacks = [ + _GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=True), tool_calls=tool_calls) + for tool_calls in (gate_tool_calls, *other_hooks_tool_calls) + ] acompletion_mock = AsyncMock() with patch.object(litellm, "acompletion", acompletion_mock): try: @@ -473,6 +478,34 @@ async def test_refused_rerun_keeps_the_clients_own_tool_call(restore_callbacks): assert result.choices[0].message.content == "Running it." +@pytest.mark.asyncio +async def test_refused_rerun_drops_every_hooks_internal_call_and_keeps_the_clients(restore_callbacks): + response = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + _tool_call("call_abc", "litellm_code_execution"), + _tool_call("call_search", "litellm_web_search"), + _tool_call("call_client", "lookup"), + ], + ), + ) + ] + ) + + result = await _run_refused_rerun( + response, + _REFUSING_RAILS[0], + other_hooks_tool_calls=({"tool_calls": [{"id": "call_search", "name": "litellm_web_search"}]},), + ) + + assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"] + + @pytest.mark.asyncio @pytest.mark.parametrize( "response, gate_tool_calls", @@ -511,6 +544,62 @@ async def test_dispatcher_surfaces_a_failed_follow_up_instead_of_the_tool_call_t assert logger.cleanup_calls == 1 +class _PlanRecordingLogger(_GateOnlyLogger): + def __init__(self, plan: AgenticLoopPlan, tool_calls: Dict[str, Any]) -> None: + super().__init__(plan=plan, tool_calls=tool_calls) + self.logging_objs: List[Any] = [] + + async def async_build_agentic_loop_plan( + self, + tools: Dict[str, Any], + model: str, + messages: List[Dict[str, Any]], + response: Any, + anthropic_messages_provider_config: Any, + anthropic_messages_optional_request_params: Dict[str, Any], + logging_obj: Any, + stream: bool, + kwargs: Dict[str, Any], + ) -> AgenticLoopPlan: + self.logging_objs.append(logging_obj) + return self._plan + + +@pytest.mark.asyncio +async def test_acompletion_bills_the_first_call_on_the_failure_log_when_the_follow_up_fails(restore_callbacks): + logger: Final = _PlanRecordingLogger( + plan=AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=_patched_messages(), kwargs={"mock_response": "litellm.RateLimitError"} + ), + ), + tool_calls={"tool_calls": [{"id": "call_abc"}]}, + ) + litellm.callbacks = [logger] + first_call: Final = ModelResponse( + choices=_tool_call_model_response().choices, usage=Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8) + ) + mock_client: Final = MagicMock() + mock_client.post = AsyncMock(return_value=_raw_response_for(first_call)) + + with pytest.raises(litellm.RateLimitError): + await litellm.acompletion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + api_key="sk-test", + client=mock_client, + num_retries=0, + ) + + (logging_obj,) = logger.logging_objs + price: Final = litellm.model_cost["gpt-4o-mini"] + assert logging_obj.model_call_details["combined_usage_object"].total_tokens == 8 + assert logging_obj.model_call_details["response_cost"] == pytest.approx( + 5 * price["input_cost_per_token"] + 3 * price["output_cost_per_token"] + ) + + class _RunOnlyLoopLogger(CustomLogger): def __init__(self, result: ModelResponse) -> None: super().__init__() diff --git a/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py index 90a6ae3a574..3bdae9ca1aa 100644 --- a/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py @@ -9,7 +9,7 @@ from unittest.mock import AsyncMock, MagicMock import pytest - +import litellm from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, @@ -21,6 +21,10 @@ from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( _handle_message_start, _parse_sse_events, ) +from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.utils import ProxyLogging +from litellm.types.integrations.custom_logger import AgenticLoopSafetyError # --------------------------------------------------------------------------- @@ -88,6 +92,14 @@ def _build_simple_text_stream() -> List[bytes]: return chunks +def _sse_frames(wire: str) -> list[tuple[str, str]]: + return [ + (frame.split("\n")[0].removeprefix("event: "), frame.split("data: ", 1)[1]) + for frame in wire.split("\n\n") + if frame.startswith("event: ") and "data: " in frame + ] + + def _build_tool_use_stream() -> List[bytes]: """Produce SSE bytes for a response with a tool_use block.""" chunks = [] @@ -727,8 +739,7 @@ class TestAgenticStreamingIteratorPhase2: class TestAgenticStreamingIteratorErrorHandling: @pytest.mark.asyncio - async def test_should_swallow_hook_errors(self): - """Errors in hook processing should be swallowed; Phase 1 chunks are still yielded.""" + async def test_a_hook_failure_reaches_the_client_after_the_streamed_message(self): chunks = _build_simple_text_stream() mock_stream = MockAsyncStream(chunks) @@ -751,11 +762,15 @@ class TestAgenticStreamingIteratorErrorHandling: ) collected = [] - async for chunk in iterator: - collected.append(chunk) - # All Phase 1 chunks should still have been yielded - assert len(collected) == len(chunks) + async def _drain(): + async for chunk in iterator: + collected.append(chunk) + + with pytest.raises(RuntimeError, match="hook exploded"): + await _drain() + + assert collected == chunks @pytest.mark.asyncio async def test_should_handle_empty_stream(self): @@ -924,12 +939,41 @@ class TestAgenticStreamingIteratorHoldBack: assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == phase2_chunks @pytest.mark.asyncio - async def test_should_error_instead_of_replaying_server_fulfilled_tool_use_when_hook_crashes(self): - """A hook crash must not replay the buffered retrieval tool_use: that is the unknown-tool bug.""" + async def test_a_rate_limited_follow_up_reaches_the_client_as_its_anthropic_error_event(self): chunks = _build_tool_use_stream() mock_handler = MagicMock() - mock_handler._call_agentic_completion_hooks = AsyncMock(side_effect=RuntimeError("hook exploded")) + mock_handler._call_agentic_completion_hooks = AsyncMock( + side_effect=litellm.RateLimitError( + message="follow-up rate limited", llm_provider="anthropic", model="claude-sonnet-4-20250514" + ) + ) + + wire = "".join( + [ + frame if isinstance(frame, str) else frame.decode() + async for frame in ProxyBaseLLMRequestProcessing.async_sse_data_generator( + response=_build_hold_back_iterator(MockAsyncStream(chunks), mock_handler), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-test"), + request_data={"model": "claude-sonnet-4-20250514"}, + proxy_logging_obj=ProxyLogging(user_api_key_cache=MagicMock()), + ) + ] + ) + + error_events = [json.loads(data) for event, data in _sse_frames(wire) if event == "error"] + assert [event["error"]["type"] for event in error_events] == ["rate_limit_error"] + assert "follow-up rate limited" in error_events[0]["error"]["message"] + assert "litellm_content_retrieve" not in wire + + @pytest.mark.asyncio + async def test_a_capped_loop_fails_the_turn_instead_of_replaying_the_retrieval_tool_use(self): + chunks = _build_tool_use_stream() + + mock_handler = MagicMock() + mock_handler._call_agentic_completion_hooks = AsyncMock( + side_effect=AgenticLoopSafetyError("Exceeded max_agentic_loops=1") + ) iterator = _build_hold_back_iterator(MockAsyncStream(chunks), mock_handler) diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index ec52c7b5bc2..164aead166b 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -3423,20 +3423,40 @@ class TestServerFulfilledToolsInRequest: == frozenset() ) - def test_interception_callbacks_declare_their_retrieval_tools(self): + @pytest.mark.parametrize("hook", ["compression", "headroom", "compresr"]) + def test_a_retrieval_hook_holds_back_a_stream_carrying_its_retrieve_tool(self, hook): from litellm.integrations.compression_interception.handler import ( LITELLM_CONTENT_RETRIEVE_TOOL_NAME, CompressionInterceptionLogger, ) + from litellm.proxy.guardrails.guardrail_hooks.compresr.compresr import ( + COMPRESR_RETRIEVE_TOOL_NAME, + CompresrGuardrail, + ) from litellm.proxy.guardrails.guardrail_hooks.headroom.headroom import ( HEADROOM_RETRIEVE_TOOL_NAME, HeadroomGuardrail, ) - assert HeadroomGuardrail.server_fulfilled_tool_names == frozenset({HEADROOM_RETRIEVE_TOOL_NAME}) - assert CompressionInterceptionLogger.server_fulfilled_tool_names == frozenset( - {LITELLM_CONTENT_RETRIEVE_TOOL_NAME} - ) + callback, retrieve_tool_name = { + "compression": (CompressionInterceptionLogger(), LITELLM_CONTENT_RETRIEVE_TOOL_NAME), + "headroom": ( + HeadroomGuardrail(api_base="https://headroom.test", api_key="hr-test", guardrail_name="headroom"), + HEADROOM_RETRIEVE_TOOL_NAME, + ), + "compresr": ( + CompresrGuardrail(api_base="https://compresr.test", api_key="cr-test", guardrail_name="compresr"), + COMPRESR_RETRIEVE_TOOL_NAME, + ), + }[hook] + tools = [ + {"name": "Bash", "input_schema": {"type": "object"}}, + {"name": retrieve_tool_name, "input_schema": {"type": "object"}}, + ] + + assert BaseLLMHTTPHandler._server_fulfilled_tools_in_request( + logging_obj=self._logging_obj_with([callback]), tools=tools + ) == frozenset({retrieve_tool_name}) def _make_stub_direct_vector_store_config(response): @@ -4994,9 +5014,11 @@ async def test_responses_agentic_followup_sends_the_plans_request_param_over_a_s class _PlannedChatLoop(CustomLogger): - def __init__(self, plan: AgenticLoopPlan) -> None: + def __init__(self, plan: AgenticLoopPlan, claimed_call_name: str = "litellm_web_search") -> None: super().__init__() self._plan = plan + self._claimed_call_name = claimed_call_name + self.logging_objs: list[object] = [] async def async_should_run_chat_completion_agentic_loop( self, @@ -5008,7 +5030,12 @@ class _PlannedChatLoop(CustomLogger): custom_llm_provider: str, kwargs: dict, ) -> tuple[bool, dict]: - return True, {"tool_calls": [{"id": "call_search", "name": "litellm_web_search"}]} + claimed: Final = [ + {"id": call.id, "name": call.function.name} + for call in _tool_calls_of(response) + if call.function.name == self._claimed_call_name + ] + return True, {"tool_calls": claimed or [{"id": "call_search", "name": self._claimed_call_name}]} async def async_build_chat_completion_agentic_loop_plan( self, @@ -5021,6 +5048,7 @@ class _PlannedChatLoop(CustomLogger): stream: bool, kwargs: dict, ) -> AgenticLoopPlan: + self.logging_objs.append(logging_obj) return self._plan @@ -5055,7 +5083,15 @@ class _RunOnlyChatLoop(CustomLogger): return self._result -async def _run_chat_completion_agentic_hooks(response: ModelResponse, *callbacks: CustomLogger) -> object: +def _tool_calls_of(response: object) -> list: + if not isinstance(response, ModelResponse) or not response.choices: + return [] + return response.choices[0].message.tool_calls or [] + + +async def _run_chat_completion_agentic_hooks( + response: ModelResponse, *callbacks: CustomLogger, kwargs: Mapping[str, object] = MappingProxyType({}) +) -> object: logging_obj: Final = Mock(litellm_call_id="call-1", dynamic_success_callbacks=list(callbacks), model_call_details={}) return await BaseLLMHTTPHandler()._call_agentic_chat_completion_hooks( response=response, @@ -5065,7 +5101,7 @@ async def _run_chat_completion_agentic_hooks(response: ModelResponse, *callbacks logging_obj=logging_obj, stream=False, custom_llm_provider="openai", - kwargs={}, + kwargs=dict(kwargs), ) @@ -5106,6 +5142,36 @@ async def test_chat_completion_agentic_hooks_return_a_run_only_loops_result(): assert await _run_chat_completion_agentic_hooks(ModelResponse(), _RunOnlyChatLoop(loop_result)) is loop_result +@pytest.mark.asyncio +async def test_chat_completion_agentic_hooks_at_the_cap_drop_every_hooks_internal_call_and_keep_the_clients(): + response: Final = ModelResponse( + choices=[ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": f"call_{name}", "type": "function", "function": {"name": name, "arguments": "{}"}} + for name in ("litellm_web_search", "litellm_code_execution", "lookup") + ], + }, + } + ] + ) + searching: Final = _PlannedChatLoop(AgenticLoopPlan(run_agentic_loop=True)) + running_code: Final = _PlannedChatLoop( + AgenticLoopPlan(run_agentic_loop=True), claimed_call_name="litellm_code_execution" + ) + + result: Final = await _run_chat_completion_agentic_hooks( + response, searching, running_code, kwargs={"_agentic_loop_depth": 1, "max_agentic_loops": 1} + ) + + assert [call.id for call in _tool_calls_of(result)] == ["call_lookup"] + + @pytest.mark.asyncio async def test_chat_completion_agentic_followup_does_not_repeat_request_params_from_plan_kwargs(monkeypatch): """A plan whose kwargs repeat a request param, or the explicitly passed model, must not crash the chat follow-up with a duplicate keyword""" @@ -5129,17 +5195,17 @@ async def test_chat_completion_agentic_followup_does_not_repeat_request_params_f ), ) - response: Final = await BaseLLMHTTPHandler()._execute_chat_completion_agentic_plan( - plan=plan, + response: Final = await BaseLLMHTTPHandler()._call_agentic_chat_completion_hooks( + response=ModelResponse(), model="gpt-5", messages=[{"role": "user", "content": "x"}], optional_params={"temperature": 0.2}, - kwargs=dict(request_kwargs), + logging_obj=Mock( + litellm_call_id="call-1", dynamic_success_callbacks=[_PlannedChatLoop(plan)], model_call_details={} + ), + stream=False, custom_llm_provider="openai", - depth=0, - max_loops=3, - fingerprints=[], - fingerprint="fp", + kwargs=dict(request_kwargs), ) assert response == "followup-response" @@ -5149,6 +5215,138 @@ async def test_chat_completion_agentic_followup_does_not_repeat_request_params_f assert followup_calls[0]["model"] == "openai/gpt-5" +def _a_search_tool_call_turn(call_number: int) -> httpx.Response: + return httpx.Response( + 200, + json={ + "id": f"chatcmpl-{call_number}", + "object": "chat.completion", + "created": 0, + "model": "served-model", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": f"call-{call_number}", + "type": "function", + "function": {"name": "litellm_web_search", "arguments": '{"query": "weather"}'}, + } + ], + }, + } + ], + "usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8}, + }, + ) + + +class _SearchAgainLoop(CustomLogger): + def __init__(self, client: AsyncHTTPHandler) -> None: + super().__init__() + self._client = client + + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: list[dict], + tools: list[dict] | None, + stream: bool, + custom_llm_provider: str, + kwargs: dict, + ) -> tuple[bool, dict]: + searches: Final = [ + {"id": call.id, "name": call.function.name} + for call in response.choices[0].message.tool_calls or [] + if call.function.name == "litellm_web_search" + ] + return bool(searches), {"tool_calls": searches} + + async def async_build_chat_completion_agentic_loop_plan( + self, + tools: dict, + model: str, + messages: list[dict], + response: object, + optional_params: dict, + logging_obj: object, + stream: bool, + kwargs: dict, + ) -> AgenticLoopPlan: + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[*messages, {"role": "user", "content": "here are the search results"}], + kwargs={"client": self._client}, + ), + ) + + +@pytest.mark.asyncio +async def test_hosted_vllm_bills_the_first_call_on_the_failure_log_when_the_follow_up_fails(monkeypatch): + loop: Final = _PlannedChatLoop( + AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[{"role": "user", "content": "here are the search results"}], + kwargs={"mock_response": "litellm.RateLimitError"}, + ), + ) + ) + monkeypatch.setattr(litellm, "callbacks", [loop]) + + with pytest.raises(litellm.RateLimitError): + await litellm.acompletion( + model="hosted_vllm/served-model", + api_base="https://fake-vllm.test/v1", + api_key="transport-only", + client=AsyncHTTPHandler(transport=httpx.MockTransport(lambda request: _a_search_tool_call_turn(1))), + messages=[{"role": "user", "content": "search the web for the weather"}], + input_cost_per_token=0.001, + output_cost_per_token=0.002, + num_retries=0, + max_retries=0, + ) + + (logging_obj,) = loop.logging_objs + assert logging_obj.model_call_details["combined_usage_object"].total_tokens == 8 + assert logging_obj.model_call_details["response_cost"] == pytest.approx(5 * 0.001 + 3 * 0.002) + + +@pytest.mark.asyncio +async def test_hosted_vllm_ends_the_turn_at_max_agentic_loops_without_the_internal_search_call(monkeypatch): + upstream_calls: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + + def always_search(request: httpx.Request) -> httpx.Response: + upstream_calls.put_nowait(request) + return _a_search_tool_call_turn(upstream_calls.qsize()) + + client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(always_search)) + monkeypatch.setattr(litellm, "callbacks", [_SearchAgainLoop(client)]) + + response: Final = await asyncio.wait_for( + litellm.acompletion( + model="hosted_vllm/served-model", + api_base="https://fake-vllm.test/v1", + api_key="transport-only", + client=client, + messages=[{"role": "user", "content": "search the web for the weather"}], + max_agentic_loops=1, + num_retries=0, + max_retries=0, + ), + timeout=10, + ) + + assert upstream_calls.qsize() == 2 + assert not response.choices[0].message.tool_calls + + _UPSTREAM_HEADERS: Final = {"x-request-id": "req_upstream", "x-ratelimit-remaining-requests": "41"} diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 9aa5a06a006..dc5795f29e2 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -19,6 +19,7 @@ import litellm from litellm import create_thread, get_thread from litellm.caching.caching import Cache, LiteLLMCacheType from litellm.integrations.custom_logger import CustomLogger +from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch from litellm.llms.openai.openai import ( AssistantEventHandler, AsyncAssistantEventHandler, @@ -246,6 +247,7 @@ class _FailingFollowUpChatLoop(_SearchingChatGate): def __init__(self, error: Exception) -> None: super().__init__() self.error = error + self.logging_objs: list[object] = [] async def async_run_chat_completion_agentic_loop( self, @@ -259,6 +261,7 @@ class _FailingFollowUpChatLoop(_SearchingChatGate): kwargs: dict, ) -> object: self.runs += 1 + self.logging_objs.append(logging_obj) raise self.error @@ -358,6 +361,25 @@ async def test_acompletion_surfaces_a_failed_agentic_follow_up_instead_of_the_fi assert loop.runs == 1 +@pytest.mark.asyncio +async def test_a_failed_agentic_follow_up_bills_the_first_call_on_the_failure_log(monkeypatch: pytest.MonkeyPatch): + loop: Final = _FailingFollowUpChatLoop( + litellm.RateLimitError(message="follow-up rate limited", llm_provider="openai", model="gpt-5.6") + ) + monkeypatch.setattr(litellm, "callbacks", [loop]) + + async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client: + with pytest.raises(litellm.RateLimitError): + await _acompletion_through(http_client) + + (logging_obj,) = loop.logging_objs + price: Final = litellm.model_cost["gpt-5.6"] + assert logging_obj.model_call_details["combined_usage_object"].total_tokens == 8 + assert logging_obj.model_call_details["response_cost"] == pytest.approx( + 5 * price["input_cost_per_token"] + 3 * price["output_cost_per_token"] + ) + + @pytest.mark.asyncio async def test_acompletion_does_not_resend_the_first_call_when_the_agentic_follow_up_answers_422( monkeypatch: pytest.MonkeyPatch, @@ -457,6 +479,78 @@ async def test_acompletion_skips_a_failing_agentic_gate_and_runs_the_next_hook(m assert response.choices[0].message.content == "after the search" +class _SearchAgainLoop(CustomLogger): + def __init__(self, client: AsyncOpenAI) -> None: + super().__init__() + self._client = client + + async def async_should_run_chat_completion_agentic_loop( + self, + response: object, + model: str, + messages: list[dict], + tools: list[dict] | None, + stream: bool, + custom_llm_provider: str, + kwargs: dict, + ) -> tuple[bool, dict]: + searches: Final = [ + {"id": call.id, "name": call.function.name} + for call in response.choices[0].message.tool_calls or [] + if call.function.name == "litellm_web_search" + ] + return bool(searches), {"tool_calls": searches} + + async def async_build_chat_completion_agentic_loop_plan( + self, + tools: dict, + model: str, + messages: list[dict], + response: object, + optional_params: dict, + logging_obj: object, + stream: bool, + kwargs: dict, + ) -> AgenticLoopPlan: + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[*messages, {"role": "user", "content": "here are the search results"}], + kwargs={"client": self._client}, + ), + ) + + +@pytest.mark.asyncio +async def test_acompletion_ends_the_turn_at_max_agentic_loops_without_the_internal_search_call( + monkeypatch: pytest.MonkeyPatch, +): + upstream_calls: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + + def always_search(request: httpx.Request) -> httpx.Response: + upstream_calls.put_nowait(request) + return _respond_with_a_search_tool_call(request) + + async with httpx.AsyncClient(transport=httpx.MockTransport(always_search)) as http_client: + client: Final = AsyncOpenAI(api_key="transport-only", http_client=http_client) + monkeypatch.setattr(litellm, "callbacks", [_SearchAgainLoop(client)]) + response: Final = await asyncio.wait_for( + litellm.acompletion( + model="openai/gpt-5.6", + api_key="transport-only", + client=client, + messages=[{"role": "user", "content": "search the web for the weather"}], + max_agentic_loops=1, + num_retries=0, + max_retries=0, + ), + timeout=10, + ) + + assert upstream_calls.qsize() == 2 + assert not response.choices[0].message.tool_calls + + _REASONING_TOOL_CALL_REPLY: Final = { "id": "chatcmpl-fake-stream", "object": "chat.completion", From b5da3b9bff6724564e4877881e1a87c50d65dafa Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:12:27 -0700 Subject: [PATCH 06/12] fix(agentic loop): bill retried attempts and clean up what failed or capped turns return The router's retry loop now carries each failed attempt's paid first call into the request's billing, so a request whose follow-up keeps failing bills every first call on its failure row, and one that recovers bills the failed attempt's first call on its success row. A failure row no longer adds unpriced estimated tokens on top of attempts that were billed, so its tokens match its spend A sandbox failure now answers the model with a fixed sentence and marks the code_interpreter_call as failed, without the sandbox's error text. Code interpreter on /v1/responses with Claude now runs its loop, because the bridged inner call no longer strips the hook's own markers Dropped internal tool calls leave the kept calls numbered from 0, also on a replayed stream. A capped native /v1/messages stream now says the loop hit its limit and that max_agentic_loops raises it, instead of asking the client to retry The agentic refusal and hook error logs no longer include the model or the exception text, and the agentic loop helpers take typed messages, params, and read-only fingerprints --- .../code_interpreter_interception/handler.py | 58 ++++--- .../agentic_loop_refusal.py | 20 ++- .../chat_completion_agentic_loop.py | 38 +++-- litellm/litellm_core_utils/litellm_logging.py | 32 ++++ .../native/agentic_streaming_iterator.py | 21 ++- litellm/llms/custom_httpx/llm_http_handler.py | 18 +-- litellm/proxy/utils.py | 4 +- litellm/router.py | 1 + litellm/types/integrations/custom_logger.py | 9 +- .../test_handler.py | 79 ++++++++-- .../test_chat_completion_agentic_loop.py | 72 +++++++++ .../native/test_agentic_streaming_iterator.py | 28 ++-- ..._utils_model_creation_and_error_logging.py | 48 ++++-- .../mcp/test_aresponses_api_with_mcp.py | 29 +--- tests/unit/test_router/test_router.py | 148 ++++++++++++++++++ 15 files changed, 487 insertions(+), 118 deletions(-) diff --git a/litellm/integrations/code_interpreter_interception/handler.py b/litellm/integrations/code_interpreter_interception/handler.py index 2cc925707ca..950088ff9e0 100644 --- a/litellm/integrations/code_interpreter_interception/handler.py +++ b/litellm/integrations/code_interpreter_interception/handler.py @@ -74,7 +74,7 @@ class CodeInterpreterLogOutput(TypedDict): class CodeInterpreterCall(TypedDict): id: str type: Literal["code_interpreter_call"] - status: Literal["completed"] + status: Literal["completed", "failed"] code: str container_id: str | None outputs: list[CodeInterpreterLogOutput] @@ -150,21 +150,27 @@ class _ReadySandbox: @dataclass(frozen=True, slots=True) class _SandboxUnavailable: - reason: str + pass _Sandbox: TypeAlias = _ReadySandbox | _SandboxUnavailable -def _unavailable_output(unavailable: _SandboxUnavailable) -> str: - return f"[sandbox unavailable] {unavailable.reason}" +@dataclass(frozen=True, slots=True) +class _ToolRun: + output: str + status: Literal["completed", "failed"] + + +_SANDBOX_UNAVAILABLE_OUTPUT: Final = "The code did not run because the code execution sandbox is unavailable." +_FAILED_TOOL_RUN: Final = _ToolRun(output=_SANDBOX_UNAVAILABLE_OUTPUT, status="failed") def _sandbox_unavailable(step: str, error: Exception) -> _SandboxUnavailable: verbose_logger.warning( "CodeInterpreterInterception: %s failed, answering the tool call with the failure: %s", step, error ) - return _SandboxUnavailable(reason=str(error)) + return _SandboxUnavailable() def _output_item_type(item: object) -> object: @@ -215,6 +221,14 @@ def _extract_identity(kwargs: Mapping[str, object]) -> str: return identity if isinstance(identity, str) else "" +def _bridged_from_responses(kwargs: Mapping[str, object], call_type: CallTypes | None) -> bool: + route: Final = getattr(kwargs.get("litellm_logging_obj"), "call_type", None) + return call_type in (CallTypes.completion, CallTypes.acompletion) and route in ( + CallTypes.responses, + CallTypes.aresponses, + ) + + def _resolve_sandbox_tool(sandbox_tool_name: str | None) -> SandboxToolParams | None: if sandbox_tool_name is None: return None @@ -287,7 +301,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): async def async_pre_call_deployment_hook( self, kwargs: dict[str, object], call_type: CallTypes | None ) -> dict | None: - if not kwargs.get("_agentic_loop_depth"): + if not kwargs.get("_agentic_loop_depth") and not _bridged_from_responses(kwargs, call_type): kwargs.pop(_INTERCEPTION_ACTIVE_KEY, None) kwargs.pop(_SANDBOX_KEY, None) self._strip_interception_metadata(kwargs) @@ -486,7 +500,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): for tool_call in tool_calls: arguments = tool_call.get("arguments", "") code = self._parse_code(arguments) - stdout = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments) + tool_run = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments) input_list.append( { "type": "function_call", @@ -499,17 +513,17 @@ class CodeInterpreterInterceptionLogger(CustomLogger): { "type": "function_call_output", "call_id": tool_call.get("call_id"), - "output": stdout, + "output": tool_run.output, } ) code_interpreter_calls.append( { "id": f"ci_{uuid.uuid4().hex}", "type": "code_interpreter_call", - "status": "completed", + "status": tool_run.status, "code": code, "container_id": container_id, - "outputs": ([{"type": "logs", "logs": stdout}] if stdout else []), + "outputs": ([{"type": "logs", "logs": tool_run.output}] if tool_run.output else []), } ) except Exception: @@ -623,21 +637,21 @@ class CodeInterpreterInterceptionLogger(CustomLogger): ) -> tuple[ChatCompletionToolMessage, CodeInterpreterCall]: arguments: Final = tool_call.get("arguments", "") code: Final = self._parse_code(arguments) - stdout: Final = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments) + tool_run: Final = await self._sandbox_tool_output(sandbox=sandbox, arguments=arguments) tool_call_id: Final = tool_call.get("id") or tool_call.get("call_id") or uuid.uuid4().hex return ( { "role": "tool", "tool_call_id": tool_call_id, - "content": stdout, + "content": tool_run.output, }, { "id": f"ci_{uuid.uuid4().hex}", "type": "code_interpreter_call", - "status": "completed", + "status": tool_run.status, "code": code, "container_id": container_id, - "outputs": [{"type": "logs", "logs": stdout}] if stdout else [], + "outputs": [{"type": "logs", "logs": tool_run.output}] if tool_run.output else [], }, ) @@ -722,14 +736,20 @@ class CodeInterpreterInterceptionLogger(CustomLogger): except (json.JSONDecodeError, TypeError, AttributeError): return "" - async def _sandbox_tool_output(self, sandbox: _Sandbox, arguments: str) -> str: + async def _sandbox_tool_output(self, sandbox: _Sandbox, arguments: str) -> _ToolRun: match sandbox: case _SandboxUnavailable(): - return _unavailable_output(sandbox) + return _FAILED_TOOL_RUN case _ReadySandbox(container=container, params=params): - return await self._run_tool_call(container=container, params=params, arguments=arguments) + match await self._run_tool_call(container=container, params=params, arguments=arguments): + case _SandboxUnavailable(): + return _FAILED_TOOL_RUN + case str() as output: + return _ToolRun(output=output, status="completed") - async def _run_tool_call(self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str) -> str: + async def _run_tool_call( + self, container: ContainerHandle, params: SandboxToolParams | None, arguments: str + ) -> str | _SandboxUnavailable: try: code: Final = json.loads(arguments).get("code", "") if arguments else "" except (json.JSONDecodeError, TypeError): @@ -738,7 +758,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): try: result: Final = await self._run_code(container=container, params=params, code=code) except Exception as e: - return _unavailable_output(_sandbox_unavailable("running the code", e)) + return _sandbox_unavailable("running the code", e) if getattr(result, "error", None): error: Final = result.error message: Final = error.get("value") or error.get("name") if isinstance(error, dict) else str(error) diff --git a/litellm/litellm_core_utils/agentic_loop_refusal.py b/litellm/litellm_core_utils/agentic_loop_refusal.py index ac3faad5f54..abd31e856f6 100644 --- a/litellm/litellm_core_utils/agentic_loop_refusal.py +++ b/litellm/litellm_core_utils/agentic_loop_refusal.py @@ -7,7 +7,12 @@ from typing import Final from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.utils import Choices, ModelResponse +from litellm.types.utils import ( + ChatCompletionMessageCustomToolCall, + ChatCompletionMessageToolCall, + Choices, + ModelResponse, +) def _strings(values: Iterable[object]) -> frozenset[str]: @@ -23,7 +28,7 @@ async def _calls_a_gate_claims(gate: Awaitable[tuple[bool, object]]) -> tuple[ob try: should_run, tool_calls = await gate except Exception as e: - verbose_logger.debug("LiteLLM.AgenticLoopRefused: skipping a gate that failed: %s", e) + verbose_logger.debug("LiteLLM.AgenticLoopRefused: skipping a gate that failed with %s", type(e).__name__) return () return _claimed_calls(tool_calls) if should_run else () @@ -73,12 +78,19 @@ def _is_refused_chat_tool_call(call: object, refused_ids: frozenset[str], refuse return is_refused_call(message_field(call, "id"), name, refused_ids, refused_names) +def _at_position( + call: ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall, position: int +) -> ChatCompletionMessageToolCall | ChatCompletionMessageCustomToolCall: + return call if message_field(call, "index") is None else call.model_copy(update={"index": position}) + + def _finalized_choice(choice: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> object: if not isinstance(choice, Choices) or not choice.message.tool_calls: return choice - kept: Final = [ + surviving: Final = ( call for call in choice.message.tool_calls if not _is_refused_chat_tool_call(call, refused_ids, refused_names) - ] + ) + kept: Final = [_at_position(call, position) for position, call in enumerate(surviving)] finish_reason: Final = "stop" if not kept and choice.finish_reason == "tool_calls" else choice.finish_reason return choice.model_copy( update={ diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 9b449f3617d..3021d417b9e 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -18,10 +18,12 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( ) from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject +from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict, object_items from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, + AgenticLoopLimitError, AgenticLoopPlan, AgenticLoopRequestPatch, AgenticLoopSafetyError, @@ -93,14 +95,14 @@ def _coerce_int(value: object, default: int) -> int: return int(value) if isinstance(value, (int, str)) else default -def _agentic_loop_settings(kwargs: Mapping[str, object]) -> tuple[int, int, list[str]]: +def _agentic_loop_settings(kwargs: Mapping[str, object]) -> tuple[int, int, tuple[str, ...]]: depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0) configured: Final = validated_max_agentic_loops( kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops" ) max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured raw_fingerprints: Final = kwargs.get("_agentic_loop_fingerprints") - fingerprints: Final = [str(fp) for fp in raw_fingerprints] if isinstance(raw_fingerprints, list) else [] + fingerprints: Final = tuple(str(fp) for fp in raw_fingerprints) if isinstance(raw_fingerprints, list) else () return depth, max_loops, fingerprints @@ -113,7 +115,7 @@ def _fingerprint_tools(tool_calls: object) -> str: def _check_agentic_loop_safety( tool_calls: object, - fingerprints: list[str], + fingerprints: Sequence[str], depth: int, max_loops: int, model: str, @@ -122,7 +124,7 @@ def _check_agentic_loop_safety( if fingerprint in fingerprints: raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun") if depth >= max_loops: - raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}") + raise AgenticLoopLimitError(f"Exceeded max_agentic_loops={max_loops} for model={model}") return fingerprint @@ -171,6 +173,10 @@ def _end_refused_turn( return finalized if replay is None else replay +def _request_tools(optional_params: Mapping[str, object]) -> list[dict[str, object]]: + return [tool for tool in object_items(optional_params.get("tools")) if is_str_keyed_dict(tool)] + + def _custom_loggers(logging_obj: object) -> tuple[CustomLogger, ...]: import litellm @@ -185,7 +191,7 @@ async def _refused_chat_turn( *, response: ModelResponse, model: str, - messages: list, + messages: list[dict[str, object]], tools: list[dict[str, object]] | None, custom_llm_provider: str, kwargs: Mapping[str, object], @@ -265,7 +271,7 @@ async def _execute_chat_completion_agentic_plan( custom_llm_provider: str, depth: int, max_loops: int, - fingerprints: list[str], + fingerprints: Sequence[str], fingerprint: str, ) -> object: import litellm @@ -341,8 +347,8 @@ async def maybe_run_chat_completion_agentic_loop( *, response: ModelResponse, model: str, - messages: list, - optional_params: dict, + messages: list[dict[str, object]], + optional_params: dict[str, object], kwargs: dict[str, object], logging_obj: object, custom_llm_provider: str, @@ -350,7 +356,7 @@ async def maybe_run_chat_completion_agentic_loop( ) -> ModelResponse | CustomStreamWrapper | None: callbacks: Final = _custom_loggers(logging_obj) depth, max_loops, fingerprints = _agentic_loop_settings(kwargs) - tools: Final = optional_params.get("tools", []) + tools: Final = _request_tools(optional_params) for callback in callbacks: if not _gate_overridden(callback): @@ -392,7 +398,7 @@ async def maybe_run_chat_completion_agentic_loop( except AgenticLoopSafetyError as e: if stream: raise - verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [model=%s]: %s", model, str(e)) + verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__) return _end_refused_turn( await _refused_chat_turn( callbacks, @@ -472,8 +478,8 @@ async def run_chat_completion_agentic_hooks( *, response: ModelResponse, model: str, - messages: list, - optional_params: dict, + messages: list[dict[str, object]], + optional_params: dict[str, object], kwargs: dict[str, object], logging_obj: LiteLLMLoggingObject, custom_llm_provider: str, @@ -481,7 +487,7 @@ async def run_chat_completion_agentic_hooks( ) -> object | None: callbacks: Final = _custom_loggers(logging_obj) depth, max_loops, fingerprints = _agentic_loop_settings(kwargs) - tools: Final = optional_params.get("tools", []) + tools: Final = _request_tools(optional_params) hook_kwargs: Final = {**kwargs, "custom_llm_provider": custom_llm_provider} for callback in callbacks: @@ -495,9 +501,9 @@ async def run_chat_completion_agentic_hooks( custom_llm_provider=custom_llm_provider, kwargs=kwargs, ) - except Exception as e: + except Exception: verbose_logger.exception( - "LiteLLM.AgenticHookError: Exception in async_should_run_chat_completion_agentic_loop: %s", str(e) + "LiteLLM.AgenticHookError: Exception in async_should_run_chat_completion_agentic_loop" ) continue @@ -515,7 +521,7 @@ async def run_chat_completion_agentic_hooks( except AgenticLoopSafetyError as e: if stream: raise - verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [model=%s]: %s", model, str(e)) + verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__) return await _refused_chat_turn( callbacks, callback, diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 3983dcc426a..3ded219f1ea 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -50,6 +50,7 @@ from litellm.constants import ( REDACTED_BY_LITELLM, ) from litellm.cost_calculator import ( + BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, get_usage_object, @@ -571,6 +572,7 @@ _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata" _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) _SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object]) _REDACTED_RESULT: Final = TypeAdapter(object) +_FAILED_ATTEMPTS_USAGE: Final = TypeAdapter(tuple[tuple[Usage, float], ...]) _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" @@ -2668,6 +2670,34 @@ class Logging(LiteLLMLoggingBaseClass): }, ) + def _add_failed_attempts_spend(self, response: object) -> None: + own_usage: Final = get_usage_object(completion_response=response) + own_cost: Final = self.model_call_details.get("response_cost") + if own_usage is None or not isinstance(own_cost, (int, float)): + return + earlier_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python( + ( + *self.model_call_details.get("failed_attempts_billed_on_success", ()), + *self.model_call_details.pop("failed_attempts_usage", ()), + ) + ) + if not earlier_attempts: + return + earlier_cost: Final = sum(cost for _, cost in earlier_attempts) + self.model_call_details["failed_attempts_billed_on_success"] = earlier_attempts + self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects( + usage_objects=[own_usage, *(usage for usage, _ in earlier_attempts)] + ) + self.model_call_details["response_cost"] = own_cost + earlier_cost + if self.cost_breakdown is not None: + self.cost_breakdown = self.cost_breakdown | CostBreakdown( + total_cost=own_cost + earlier_cost, + additional_costs={ + **self.cost_breakdown.get("additional_costs", {}), + "failed_attempts_cost": earlier_cost, + }, + ) + def claim_stream_failure_report(self) -> bool: if self._stream_failure_claimed: return False @@ -2925,6 +2955,7 @@ class Logging(LiteLLMLoggingBaseClass): elif "response_cost" in hidden_params: self.model_call_details["response_cost"] = hidden_params["response_cost"] self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"]) + self._add_failed_attempts_spend(logging_result) elif (existing_cost := self.model_call_details.get("response_cost")) is not None and ( existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped ): @@ -2934,6 +2965,7 @@ class Logging(LiteLLMLoggingBaseClass): pass else: self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result) + self._add_failed_attempts_spend(logging_result) if not build_logging_payload: return None diff --git a/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py b/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py index c33a849b5b3..0aa30998557 100644 --- a/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py +++ b/litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py @@ -22,7 +22,7 @@ from typing import TYPE_CHECKING, Any, Final, cast from litellm._logging import verbose_logger from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES -from litellm.types.integrations.custom_logger import AgenticLoopSafetyError +from litellm.types.integrations.custom_logger import AgenticLoopLimitError, AgenticLoopSafetyError if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -36,10 +36,16 @@ SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES: Final = ( b'data: {"type": "error", "error": {"type": "api_error", "message": ' b'"Server-side tool retrieval failed, so this turn could not be completed. Please retry."}}\n\n' ) +AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES: Final = ( + b"event: error\n" + b'data: {"type": "error", "error": {"type": "api_error", "message": ' + b'"The agentic loop reached its limit before this turn finished. ' + b'Raise max_agentic_loops to allow more loops."}}\n\n' +) def is_server_fulfilled_tool_leak_error(chunk: object) -> bool: - return chunk == SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES + return chunk in (SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES) async def _anext_or_none(iterator: AsyncIterator) -> bytes | None: @@ -217,6 +223,7 @@ class AgenticAnthropicStreamingIterator: self._follow_up_chunk_task: asyncio.Task | None = None self._replay_index = 0 self._error_emitted = False + self._loop_limit_reached = False @property def has_buffered_provider_output(self) -> bool: @@ -289,6 +296,8 @@ class AgenticAnthropicStreamingIterator: "tool_use [model=%s]; emitting an SSE error instead of leaking the tool call to the client", self._model, ) + if self._loop_limit_reached: + return AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES return SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES if self._replay_index < len(self._collected_bytes): @@ -395,12 +404,8 @@ class AgenticAnthropicStreamingIterator: type(result).__name__, ) except AgenticLoopSafetyError as e: - verbose_logger.warning( - "AgenticStreamingIterator: agentic loop refused [call_id=%s model=%s]: %s", - getattr(self._logging_obj, "litellm_call_id", "unknown"), - self._model, - str(e), - ) + self._loop_limit_reached = isinstance(e, AgenticLoopLimitError) + verbose_logger.warning("AgenticStreamingIterator: agentic loop refused [rail=%s]", type(e).__name__) @staticmethod def _rebuild_anthropic_response_from_sse( diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 65f19df2a9d..5b74dd60068 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -149,6 +149,7 @@ from litellm.types.extraction import ExtractionResponse from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig from litellm.types.integrations.custom_logger import ( NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, + AgenticLoopLimitError, AgenticLoopPlan, AgenticLoopRequestPatch, AgenticLoopSafetyError, @@ -5967,7 +5968,7 @@ class BaseLLMHTTPHandler: @staticmethod def _check_agentic_loop_safety( tool_calls: object, - fingerprints: list[str], + fingerprints: Sequence[str], depth: int, max_loops: int, model: str, @@ -5986,7 +5987,7 @@ class BaseLLMHTTPHandler: if fingerprint in fingerprints: raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun") if depth >= max_loops: - raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}") + raise AgenticLoopLimitError(f"Exceeded max_agentic_loops={max_loops} for model={model}") return fingerprint @staticmethod @@ -6070,7 +6071,7 @@ class BaseLLMHTTPHandler: kwargs: dict, depth: int, max_loops: int, - fingerprints: list[str], + fingerprints: Sequence[str], fingerprint: str, stream: bool = False, callback: Optional["CustomLogger"] = None, @@ -6111,7 +6112,7 @@ class BaseLLMHTTPHandler: kwargs_for_followup.update(patch.kwargs) kwargs_for_followup["_agentic_loop_depth"] = depth + 1 kwargs_for_followup["max_agentic_loops"] = max_loops - kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint] + kwargs_for_followup["_agentic_loop_fingerprints"] = [*fingerprints, fingerprint] response: AnthropicMessagesResponse | AsyncIterator[object] = await anthropic_messages.acreate( **{ @@ -6150,7 +6151,7 @@ class BaseLLMHTTPHandler: kwargs: dict, depth: int, max_loops: int, - fingerprints: list[str], + fingerprints: Sequence[str], fingerprint: str, callback: Optional["CustomLogger"] = None, ) -> ResponsesAPIResponse | BaseResponsesAPIStreamingIterator: @@ -6386,12 +6387,7 @@ class BaseLLMHTTPHandler: except AgenticLoopSafetyError as e: if not self._can_replace_turn_with_terminal_response(stream, api_surface): raise - verbose_logger.warning( - "LiteLLM.AgenticLoopRefused: ending turn [call_id=%s model=%s]: %s", - logging_obj.litellm_call_id, - model, - str(e), - ) + verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [rail=%s]", type(e).__name__) return self._maybe_wrap_in_fake_stream( self._finalize_refused_agentic_response( response=response, diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index ddaa8d2e69d..3e905363102 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1359,16 +1359,16 @@ def _failure_usage_to_lift( base_model: str | None = None, dispatched_call_type: str | None = None, ) -> tuple[object, object] | None: + failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ())) current_attempt: Final = _current_failure_usage_to_lift( model_call_details, request_body, - dispatched, + dispatched and (provider_billed or not failed_attempts), provider_billed, deployment_id, base_model, dispatched_call_type=dispatched_call_type, ) - failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ())) if not failed_attempts: return current_attempt current_usage, current_cost = current_attempt if current_attempt is not None else (None, None) diff --git a/litellm/router.py b/litellm/router.py index 233e7581a40..14879ae78b5 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -9432,6 +9432,7 @@ class Router(DeprecatedRouterMembers): # Update retry tracking metadata before each retry attempt _metadata["attempted_retries"] = current_attempt + 1 _metadata["max_retries"] = num_retries + carry_superseded_attempt_usage(kwargs) # if the function call is successful, no exception will be raised and we'll break out of the loop response = await self.make_call(original_function, *args, **kwargs) if coroutine_checker.is_async_callable(response): # async errors are often returned as coroutines diff --git a/litellm/types/integrations/custom_logger.py b/litellm/types/integrations/custom_logger.py index 77e75dea030..14ed719a880 100644 --- a/litellm/types/integrations/custom_logger.py +++ b/litellm/types/integrations/custom_logger.py @@ -74,14 +74,13 @@ class AgenticLoopSafetyError(ValueError): Covers both rails: the bounded-loop cap (``max_agentic_loops``) and the repeated tool-call fingerprint cycle break. Subclasses ``ValueError`` so callers that already catch the broader type keep working. - - Only the anthropic messages loop raises this today. The chat completions - loop in ``litellm_core_utils/chat_completion_agentic_loop.py`` still raises - a plain ``ValueError`` from its own copy of the same rails, so catching - this type alone will not cover that surface until it is moved over. """ +class AgenticLoopLimitError(AgenticLoopSafetyError): + pass + + class StandardCustomLoggerInitParams(LiteLLMBaseModel): """ Params for initializing a CustomLogger. diff --git a/tests/unit/integrations/code_interpreter_interception/test_handler.py b/tests/unit/integrations/code_interpreter_interception/test_handler.py index 6f94e845e69..1c988a35fdc 100644 --- a/tests/unit/integrations/code_interpreter_interception/test_handler.py +++ b/tests/unit/integrations/code_interpreter_interception/test_handler.py @@ -8,13 +8,18 @@ config and records how it is called. import time from typing import Final +import httpx import pytest +import respx + +import litellm from litellm.integrations.code_interpreter_interception.handler import ( _INTERCEPTION_ACTIVE_KEY as _ACTIVE_KEY, ) from litellm.integrations.code_interpreter_interception.handler import ( _SANDBOX_KEY, + _SANDBOX_UNAVAILABLE_OUTPUT, _SESSION_SCOPED_KEY, LITELLM_CODE_EXECUTION_TOOL_NAME, CodeInterpreterInterceptionLogger, @@ -1455,25 +1460,26 @@ class _SandboxThatCannotRunCode(FakeSandbox): _SANDBOX_FAILURES: Final = ( - pytest.param(None, "[sandbox unavailable] CodeInterpreterInterception: no sandbox available", id="no_sandbox"), - pytest.param(_SandboxThatCannotStart(), "[sandbox unavailable] sandbox provider unreachable", id="create_fails"), - pytest.param(_SandboxThatCannotRunCode(), "[sandbox unavailable] sandbox run timed out", id="run_fails"), + pytest.param(None, id="no_sandbox"), + pytest.param(_SandboxThatCannotStart(), id="create_fails"), + pytest.param(_SandboxThatCannotRunCode(), id="run_fails"), ) @pytest.mark.asyncio -@pytest.mark.parametrize(("sandbox", "expected_output"), _SANDBOX_FAILURES) -async def test_a_sandbox_failure_answers_the_responses_tool_call_with_the_failure(sandbox, expected_output): +@pytest.mark.parametrize("sandbox", _SANDBOX_FAILURES) +async def test_a_sandbox_failure_answers_the_responses_tool_call_with_a_fixed_failure(sandbox): plan: Final = await _build_plan(CodeInterpreterInterceptionLogger(sandbox_config=sandbox), sandbox) outputs: Final = [message for message in _iter_messages(plan) if message.get("type") == "function_call_output"] assert plan.run_agentic_loop is True - assert [(output["call_id"], output["output"].startswith(expected_output)) for output in outputs] == [("c1", True)] + assert [(output["call_id"], output["output"]) for output in outputs] == [("c1", _SANDBOX_UNAVAILABLE_OUTPUT)] + assert [call["status"] for call in plan.metadata["code_interpreter_calls"]] == ["failed"] @pytest.mark.asyncio -@pytest.mark.parametrize(("sandbox", "expected_output"), _SANDBOX_FAILURES) -async def test_a_sandbox_failure_answers_the_chat_tool_call_with_the_failure(sandbox, expected_output): +@pytest.mark.parametrize("sandbox", _SANDBOX_FAILURES) +async def test_a_sandbox_failure_answers_the_chat_tool_call_with_a_fixed_failure(sandbox): logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) plan: Final = await logger.async_build_agentic_loop_plan( @@ -1499,6 +1505,59 @@ async def test_a_sandbox_failure_answers_the_chat_tool_call_with_the_failure(san tool_messages: Final = [message for message in _iter_messages(plan) if message.get("role") == "tool"] assert plan.run_agentic_loop is True - assert [(message["tool_call_id"], message["content"].startswith(expected_output)) for message in tool_messages] == [ - ("call_1", True) + assert [(message["tool_call_id"], message["content"]) for message in tool_messages] == [ + ("call_1", _SANDBOX_UNAVAILABLE_OUTPUT) ] + assert [call["status"] for call in plan.metadata["code_interpreter_calls"]] == ["failed"] + + +def _anthropic_message(content: list[dict[str, object]], stop_reason: str) -> dict[str, object]: + return { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": content, + "stop_reason": stop_reason, + "stop_sequence": None, + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + + +@pytest.mark.asyncio +async def test_a_responses_request_bridged_to_claude_runs_the_code_and_ends_without_the_internal_call( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + sandbox: Final = FakeSandbox(stdout="42") + monkeypatch.setattr(litellm, "callbacks", [CodeInterpreterInterceptionLogger(sandbox_config=sandbox)]) + provider: Final = respx_mock.post("https://api.anthropic.com/v1/messages").mock( + side_effect=[ + httpx.Response( + 200, + json=_anthropic_message( + [ + { + "type": "tool_use", + "id": "toolu_1", + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "input": {"code": "print(40 + 2)"}, + } + ], + "tool_use", + ), + ), + httpx.Response(200, json=_anthropic_message([{"type": "text", "text": "The answer is 42."}], "end_turn")), + ] + ) + + response: Final = await litellm.aresponses( + model="anthropic/claude-sonnet-5-5", + input="compute 40 + 2", + api_key="sk-test", + tools=[{"type": "code_interpreter", "container": {"type": "auto"}}], + ) + + assert provider.call_count == 2 + assert [run["code"] for run in sandbox.run_calls] == ["print(40 + 2)"] + assert not [item for item in response.output if getattr(item, "type", None) == "function_call"] diff --git a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py index 745374ab431..ac3707adf13 100644 --- a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -522,6 +522,78 @@ async def test_refused_rerun_keeps_the_clients_own_tool_call(restore_callbacks): assert result.choices[0].message.content == "Running it." +def _indexed_tool_call(call_id: str, name: str, index: int) -> ChatCompletionMessageToolCall: + return ChatCompletionMessageToolCall( + id=call_id, type="function", function=Function(name=name, arguments="{}"), index=index + ) + + +def _turn_with_the_internal_call_between_the_clients() -> ModelResponse: + return ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + _indexed_tool_call("call_weather", "get_weather", 0), + _indexed_tool_call("call_abc", "litellm_code_execution", 1), + _indexed_tool_call("call_time", "get_time", 2), + ], + ), + ) + ] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("rail_kwargs", _REFUSING_RAILS, ids=["max_agentic_loops", "repeated_fingerprint"]) +async def test_refused_rerun_renumbers_the_kept_tool_calls_from_zero(restore_callbacks, rail_kwargs: Dict[str, object]): + result = await _run_refused_rerun(_turn_with_the_internal_call_between_the_clients(), rail_kwargs) + + assert [(call.id, call.index) for call in result.choices[0].message.tool_calls] == [ + ("call_weather", 0), + ("call_time", 1), + ] + + +@pytest.mark.asyncio +async def test_refused_rerun_on_a_converted_stream_replays_the_kept_tool_calls_from_index_zero( + restore_callbacks, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr( + litellm, + "callbacks", + [_GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=True), tool_calls=_REFUSED_GATE_TOOL_CALLS)], + ) + logging_obj: Final = Logging( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-refused-stream", + function_id="fn-refused-stream", + ) + + result: Final = await maybe_run_chat_completion_agentic_loop( + response=_turn_with_the_internal_call_between_the_clients(), + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + optional_params={}, + kwargs={**_REFUSING_RAILS[1], f"{CODE_INTERPRETER_INTERCEPTION_PREFIX}_converted_stream": True}, + logging_obj=logging_obj, + custom_llm_provider="openai", + stream=False, + ) + + assert isinstance(result, CustomStreamWrapper) + deltas: Final = [chunk.choices[0].delta async for chunk in result if chunk.choices] + streamed_calls: Final = [call for delta in deltas for call in delta.tool_calls or []] + assert [(call.id, call.index) for call in streamed_calls] == [("call_weather", 0), ("call_time", 1)] + + @pytest.mark.asyncio async def test_refused_rerun_drops_every_hooks_internal_call_and_keeps_the_clients(restore_callbacks): response = ModelResponse( diff --git a/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py b/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py index 3bdae9ca1aa..7db81b7841e 100644 --- a/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py +++ b/tests/unit/llms/anthropic/messages/native/test_agentic_streaming_iterator.py @@ -21,10 +21,10 @@ from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( _handle_message_start, _parse_sse_events, ) +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.utils import ProxyLogging -from litellm.types.integrations.custom_logger import AgenticLoopSafetyError # --------------------------------------------------------------------------- @@ -967,22 +967,26 @@ class TestAgenticStreamingIteratorHoldBack: assert "litellm_content_retrieve" not in wire @pytest.mark.asyncio - async def test_a_capped_loop_fails_the_turn_instead_of_replaying_the_retrieval_tool_use(self): - chunks = _build_tool_use_stream() - + async def test_a_capped_loop_tells_the_client_to_raise_max_agentic_loops_instead_of_retrying(self): mock_handler = MagicMock() mock_handler._call_agentic_completion_hooks = AsyncMock( - side_effect=AgenticLoopSafetyError("Exceeded max_agentic_loops=1") + side_effect=lambda **_: BaseLLMHTTPHandler._check_agentic_loop_safety( + tool_calls=[{"name": "litellm_content_retrieve"}], + fingerprints=[], + depth=1, + max_loops=1, + model="claude-sonnet-4-20250514", + ) ) - iterator = _build_hold_back_iterator(MockAsyncStream(chunks), mock_handler) + iterator = _build_hold_back_iterator(MockAsyncStream(_build_tool_use_stream()), mock_handler) + wire = b"".join([chunk async for chunk in iterator if chunk != STREAM_SSE_KEEPALIVE_PING_BYTES]).decode() - collected = [] - async for chunk in iterator: - collected.append(chunk) - - assert [c for c in collected if c != STREAM_SSE_KEEPALIVE_PING_BYTES] == [SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES] - assert b"litellm_content_retrieve" not in b"".join(collected) + error_messages = [json.loads(data)["error"]["message"] for event, data in _sse_frames(wire) if event == "error"] + assert len(error_messages) == 1 + assert "max_agentic_loops" in error_messages[0] + assert "retry" not in error_messages[0].lower() + assert "litellm_content_retrieve" not in wire @pytest.mark.asyncio async def test_should_error_instead_of_replaying_when_no_hook_fires_on_tool_use(self): diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 3cdd2edb2fb..291231a4aaf 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -3561,15 +3561,13 @@ def test_an_earlier_timed_out_responses_attempt_counts_its_input_from_the_newest [None, 1700000001.0], ids=["retry-not-dispatched", "retry-dispatched"], ) -def test_a_retry_failure_bills_input_only_for_dispatched_attempts( +def test_an_unbilled_retry_failure_adds_no_estimated_tokens_to_the_billed_attempts( current_attempt_api_call_start_time: float | None, ) -> None: - from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift + from litellm.proxy.utils import _failure_fields_to_lift from litellm.types.utils import Usage prior_usage: Final = Usage(prompt_tokens=4, completion_tokens=2, total_tokens=6) - estimated_retry_usage: Final = _estimate_dispatched_failure_usage(FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None) - assert estimated_retry_usage is not None lifted: Final = _failure_fields_to_lift( { @@ -3588,15 +3586,45 @@ def test_a_retry_failure_bills_input_only_for_dispatched_attempts( usage: Final = lifted["combined_usage_object"] assert isinstance(usage, Usage) - expected_retry_tokens: Final = ( - estimated_retry_usage.prompt_tokens if current_attempt_api_call_start_time is not None else 0 - ) - assert usage.prompt_tokens == prior_usage.prompt_tokens + expected_retry_tokens - assert usage.completion_tokens == prior_usage.completion_tokens - assert usage.total_tokens == prior_usage.total_tokens + expected_retry_tokens + assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (4, 2, 6) assert lifted["response_cost"] == 0.25 +def test_a_responses_failure_after_a_retried_follow_up_bills_only_the_first_calls_usage() -> None: + from litellm.litellm_core_utils.litellm_logging import Logging + from litellm.proxy.utils import _failure_fields_to_lift + from litellm.types.utils import ModelResponse, Usage + + responses_input: Final = [{"role": "user", "content": "Search the web for the latest LiteLLM release"}] + logging_obj: Final = Logging( + model=FAILURE_USAGE_MODEL, + messages=responses_input, + stream=False, + call_type="aresponses", + start_time=real_datetime.datetime.now(), + litellm_call_id="call-responses-retry", + function_id="fn-responses-retry", + ) + logging_obj.update_environment_variables( + model=FAILURE_USAGE_MODEL, user="", optional_params={}, litellm_params={"custom_llm_provider": "openai"} + ) + logging_obj.pre_call(input=responses_input, api_key="sk-test") + first_call: Final = ModelResponse(usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672)) + logging_obj.record_earlier_round_for_failure(first_call) + first_call_cost: Final = logging_obj.model_call_details["response_cost"] + logging_obj.carry_failed_attempt_usage() + logging_obj.pre_call(input=responses_input, api_key="sk-test") + + lifted: Final = _failure_fields_to_lift( + {"litellm_logging_obj": logging_obj}, provider_billed=False, dispatched_call_type="aresponses" + ) + + usage: Final = lifted["combined_usage_object"] + assert (usage.prompt_tokens, usage.completion_tokens) == (559, 113) + assert first_call_cost > 0 + assert lifted["response_cost"] == pytest.approx(first_call_cost) + + @pytest.mark.parametrize( "model_call_details, dispatched", [ diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index ec0be896095..d682fef85c3 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -26,7 +26,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.xai.common_utils import xai_reported_cost_in_usd from litellm.proxy.mcp_server import mcp_server_manager from litellm.proxy.mcp_server.faults.list_outcomes import ServerListOk -from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift +from litellm.proxy.utils import _failure_fields_to_lift from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler from litellm.responses.mcp.mcp_streaming_iterator import MCPEnhancedStreamingIterator from litellm.types.llms.openai import ( @@ -1918,7 +1918,7 @@ def _router_retrying_once() -> litellm.Router: @pytest.mark.asyncio -async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_each_attempt_its_own_rounds( +async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_the_failed_attempts_first_round_on_success( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ): _serve_responses( @@ -1945,9 +1945,11 @@ async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_ea assert len(recorder.logged) == 1 row: Final = recorder.logged[0] assert row["id"] == response.id - assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149, 40 + 5) - assert (row["prompt_tokens"], row["completion_tokens"]) == (response.usage.input_tokens, response.usage.output_tokens) - assert row["response_cost"] == pytest.approx(_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY)) + assert (response.usage.input_tokens, response.usage.output_tokens) == (500 + 149, 40 + 5) + assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149 + 71, 40 + 5 + 19) + assert row["response_cost"] == pytest.approx( + _round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY) + _round_cost(_ROUND_ONE_BODY) + ) @pytest.mark.asyncio @@ -1982,22 +1984,7 @@ async def test_non_stream_auto_execute_failing_every_retry_bills_each_paid_first failure_row_fields: Final = _failure_fields_to_lift(MappingProxyType({"litellm_logging_obj": logging_obj})) billed_usage: Final = failure_row_fields["combined_usage_object"] assert isinstance(billed_usage, Usage) - estimated_failure_usage: Final = ( - _estimate_dispatched_failure_usage( - model="gpt-5.6", - request_input=logging_obj.model_call_details.get("messages"), - system_input=None, - ) - if not retried_first_rounds - else None - ) - expected_prompt_tokens: Final = billed_tokens[0] + ( - estimated_failure_usage.prompt_tokens if estimated_failure_usage is not None else 0 - ) - assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == ( - expected_prompt_tokens, - billed_tokens[1], - ) + assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == billed_tokens assert failure_row_fields["response_cost"] == pytest.approx( _round_cost(_ROUND_ONE_BODY) + sum(_round_cost(body) for body in retried_first_rounds) ) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index a6436e47507..6651832520c 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -42,6 +42,7 @@ from litellm.integrations.otel.plumbing.context import set_request_root_span from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.llms.anthropic.messages.native.streaming_iterator import closing_superseded_attempt from litellm.llms.anthropic.messages.native.agentic_streaming_iterator import ( + AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES, SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES, ) from litellm.llms.bedrock.common_utils import BedrockError @@ -81,6 +82,7 @@ from litellm.router_utils.fallback_event_handlers import ( from litellm.router_utils.prompt_caching_cache import PromptCachingCache from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute from litellm.scheduler import FlowItem +from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest from litellm.types.router import ( CustomRoutingStrategyBase, @@ -3843,6 +3845,151 @@ async def test_continued_stream_sharing_the_proxy_logging_object_bills_the_cut_t ) +_AGENTIC_FIRST_CALL_USAGE: Final = {"prompt_tokens": 50, "completion_tokens": 7, "total_tokens": 57} +_AGENTIC_FIRST_CALL_BODY: Final = { + "id": "chatcmpl-first", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_abc", "type": "function", "function": {"name": "litellm_code_execution", "arguments": "{}"}} + ], + }, + } + ], + "usage": _AGENTIC_FIRST_CALL_USAGE, +} + + +class _ScriptedAgenticFollowUps(CustomLogger): + def __init__(self, follow_ups: tuple[str, ...]) -> None: + super().__init__() + self._follow_ups: Final = follow_ups + self.plans_built = 0 + self.shared_call_succeeded: Final = asyncio.Event() + + async def async_should_run_agentic_loop( + self, response, model, messages, tools, stream, custom_llm_provider, kwargs + ) -> tuple[bool, dict[str, object]]: + return bool(response.choices[0].message.tool_calls), {"tool_calls": [{"id": "call_abc"}]} + + async def async_build_agentic_loop_plan( + self, + tools, + model, + messages, + response, + anthropic_messages_provider_config, + anthropic_messages_optional_request_params, + logging_obj, + stream, + kwargs, + ) -> AgenticLoopPlan: + follow_up: Final = self._follow_ups[min(self.plans_built, len(self._follow_ups) - 1)] + self.plans_built += 1 + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[{"role": "user", "content": "what is 6*7?"}, {"role": "assistant", "content": "42"}], + kwargs={"mock_response": follow_up}, + ), + ) + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + if kwargs.get("litellm_call_id") == "shared-agentic-call": + self.shared_call_succeeded.set() + + +async def _route_agentic_request_with_retries( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...] +) -> tuple[object, LiteLLMLogging, respx.Route, _ScriptedAgenticFollowUps]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + hook: Final = _ScriptedAgenticFollowUps(follow_ups) + monkeypatch.setattr(litellm, "callbacks", [hook]) + first_calls: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY) + ) + router: Final = Router( + model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}], + num_retries=2, + retry_after=0, + ) + messages: Final = [{"role": "user", "content": "what is 6*7?"}] + shared_logging, _ = litellm.utils.function_setup( + original_function="acompletion", + rules_obj=litellm.utils.Rules(), + start_time=datetime.now(), + model="m", + messages=messages, + litellm_call_id="shared-agentic-call", + ) + try: + outcome: object = await router.acompletion( + model="m", messages=messages, litellm_call_id="shared-agentic-call", litellm_logging_obj=shared_logging + ) + except litellm.RateLimitError as error: + outcome = error + return outcome, shared_logging, first_calls, hook + + +@pytest.mark.asyncio +async def test_retried_agentic_request_whose_follow_up_always_fails_bills_every_first_call( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + from litellm.proxy.utils import _failure_fields_to_lift + + outcome, shared_logging, first_calls, _ = await _route_agentic_request_with_retries( + monkeypatch, respx_mock, ("litellm.RateLimitError",) + ) + + lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": shared_logging}) + first_call_cost: Final = sum( + litellm.cost_per_token(model="gpt-4o-mini", prompt_tokens=50, completion_tokens=7) + ) + assert isinstance(outcome, litellm.RateLimitError) + assert first_calls.call_count == 3 + assert lifted["combined_usage_object"].prompt_tokens == 3 * _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"] + assert lifted["combined_usage_object"].completion_tokens == 3 * _AGENTIC_FIRST_CALL_USAGE["completion_tokens"] + assert lifted["response_cost"] == pytest.approx(3 * first_call_cost) + + +@pytest.mark.asyncio +async def test_retried_agentic_request_that_recovers_bills_the_failed_attempts_first_call_on_success( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + outcome, shared_logging, first_calls, hook = await _route_agentic_request_with_retries( + monkeypatch, respx_mock, ("litellm.RateLimitError", "42") + ) + await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10) + + assert isinstance(outcome, litellm.ModelResponse) + success: Final = shared_logging.model_call_details["standard_logging_object"] + follow_up_cost: Final = sum( + litellm.cost_per_token( + model="gpt-4o-mini", + prompt_tokens=outcome.usage.prompt_tokens, + completion_tokens=outcome.usage.completion_tokens, + ) + ) + failed_first_call_cost: Final = sum( + litellm.cost_per_token(model="gpt-4o-mini", prompt_tokens=50, completion_tokens=7) + ) + assert first_calls.call_count == 2 + assert success["prompt_tokens"] == outcome.usage.prompt_tokens + _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"] + assert ( + success["completion_tokens"] + == outcome.usage.completion_tokens + _AGENTIC_FIRST_CALL_USAGE["completion_tokens"] + ) + assert success["response_cost"] == pytest.approx(follow_up_cost + failed_first_call_cost) + + def _responses_sse(*events: Mapping[str, object]) -> str: return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) @@ -18520,6 +18667,7 @@ def test_anthropic_stream_forwards_ping_live_direct_call(): def test_anthropic_stream_error_is_gateway_verdict_direct_call(): assert _anthropic_stream_error_is_gateway_verdict(SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES) is True + assert _anthropic_stream_error_is_gateway_verdict(AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES) is True assert _anthropic_stream_error_is_gateway_verdict(_anthropic_messages_overloaded_error_chunk()) is False assert _anthropic_stream_error_is_gateway_verdict(_anthropic_messages_ping_chunk()) is False From 4df38d14aa1971e3ce841b443a41703dc52ea2d5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 30 Oct 2026 19:35:12 -0700 Subject: [PATCH 07/12] fix(agentic loop): keep a recovered retry's cost breakdown on the response it returns A retried agentic request that recovers no longer adds the failed attempt's first call to its success row. The proxy keeps the follow-up's own row for that request id, so the added spend never showed up in the stored row, and that duplicate row is tracked separately. When every attempt fails, the failure row still bills each attempt's first call Pricing a first call for the failure row no longer replaces the request's cost breakdown. Before, a recovered retry answered with x-litellm-response-cost-original, -input, and -output from the failed attempt's first call. Now its headers and logged breakdown match a request that succeeds on its first try --- litellm/litellm_core_utils/litellm_logging.py | 46 +++++------------- .../mcp/test_aresponses_api_with_mcp.py | 10 ++-- tests/unit/test_router/test_router.py | 47 ++++++++++--------- 3 files changed, 40 insertions(+), 63 deletions(-) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 3ded219f1ea..a1257340d59 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -50,7 +50,6 @@ from litellm.constants import ( REDACTED_BY_LITELLM, ) from litellm.cost_calculator import ( - BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, get_usage_object, @@ -572,7 +571,6 @@ _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata" _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) _SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object]) _REDACTED_RESULT: Final = TypeAdapter(object) -_FAILED_ATTEMPTS_USAGE: Final = TypeAdapter(tuple[tuple[Usage, float], ...]) _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" @@ -2234,12 +2232,19 @@ class Logging(LiteLLMLoggingBaseClass): return 0.0 def uncached_price(self, result: object) -> UncachedPrice: + return self._price_without_recording(result, cache_hit=False, ignore_stamped_cost=True) + + def _price_without_recording( + self, result: object, *, cache_hit: bool | None, ignore_stamped_cost: bool + ) -> UncachedPrice: previous_cost_breakdown: Final = self.cost_breakdown previous_billed_token_rates: Final = self.billed_token_rates debug_key: Final = "response_cost_failure_debug_information" previous_failure_debug_info: Final = self.model_call_details.get(debug_key) try: - cost: Final = self.response_cost_calculator(result=result, cache_hit=False, ignore_stamped_cost=True) + cost: Final = self.response_cost_calculator( + result=result, cache_hit=cache_hit, ignore_stamped_cost=ignore_stamped_cost + ) return UncachedPrice(cost=cost, cost_breakdown=self.cost_breakdown) finally: self.cost_breakdown = previous_cost_breakdown @@ -2670,34 +2675,6 @@ class Logging(LiteLLMLoggingBaseClass): }, ) - def _add_failed_attempts_spend(self, response: object) -> None: - own_usage: Final = get_usage_object(completion_response=response) - own_cost: Final = self.model_call_details.get("response_cost") - if own_usage is None or not isinstance(own_cost, (int, float)): - return - earlier_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python( - ( - *self.model_call_details.get("failed_attempts_billed_on_success", ()), - *self.model_call_details.pop("failed_attempts_usage", ()), - ) - ) - if not earlier_attempts: - return - earlier_cost: Final = sum(cost for _, cost in earlier_attempts) - self.model_call_details["failed_attempts_billed_on_success"] = earlier_attempts - self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects( - usage_objects=[own_usage, *(usage for usage, _ in earlier_attempts)] - ) - self.model_call_details["response_cost"] = own_cost + earlier_cost - if self.cost_breakdown is not None: - self.cost_breakdown = self.cost_breakdown | CostBreakdown( - total_cost=own_cost + earlier_cost, - additional_costs={ - **self.cost_breakdown.get("additional_costs", {}), - "failed_attempts_cost": earlier_cost, - }, - ) - def claim_stream_failure_report(self) -> bool: if self._stream_failure_claimed: return False @@ -2707,7 +2684,10 @@ class Logging(LiteLLMLoggingBaseClass): def record_earlier_round_for_failure(self, response: object) -> None: usage: Final = get_usage_object(completion_response=response) if usage is not None: - self.record_partial_usage_for_failure(usage, self.response_cost_calculator(result=response) or 0.0) + earlier_round_price: Final = self._price_without_recording( + response, cache_hit=None, ignore_stamped_cost=False + ) + self.record_partial_usage_for_failure(usage, earlier_round_price.cost or 0.0) def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None: """Bill a fully streamed response on the failure log when a post-call hook rejects it.""" @@ -2955,7 +2935,6 @@ class Logging(LiteLLMLoggingBaseClass): elif "response_cost" in hidden_params: self.model_call_details["response_cost"] = hidden_params["response_cost"] self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"]) - self._add_failed_attempts_spend(logging_result) elif (existing_cost := self.model_call_details.get("response_cost")) is not None and ( existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped ): @@ -2965,7 +2944,6 @@ class Logging(LiteLLMLoggingBaseClass): pass else: self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result) - self._add_failed_attempts_spend(logging_result) if not build_logging_payload: return None diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index d682fef85c3..af9a0b50805 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -1918,7 +1918,7 @@ def _router_retrying_once() -> litellm.Router: @pytest.mark.asyncio -async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_the_failed_attempts_first_round_on_success( +async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_each_attempt_its_own_rounds( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ): _serve_responses( @@ -1945,11 +1945,9 @@ async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_th assert len(recorder.logged) == 1 row: Final = recorder.logged[0] assert row["id"] == response.id - assert (response.usage.input_tokens, response.usage.output_tokens) == (500 + 149, 40 + 5) - assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149 + 71, 40 + 5 + 19) - assert row["response_cost"] == pytest.approx( - _round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY) + _round_cost(_ROUND_ONE_BODY) - ) + assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149, 40 + 5) + assert (row["prompt_tokens"], row["completion_tokens"]) == (response.usage.input_tokens, response.usage.output_tokens) + assert row["response_cost"] == pytest.approx(_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY)) @pytest.mark.asyncio diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index 6651832520c..dc7e8d64889 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -3960,34 +3960,35 @@ async def test_retried_agentic_request_whose_follow_up_always_fails_bills_every_ assert lifted["response_cost"] == pytest.approx(3 * first_call_cost) -@pytest.mark.asyncio -async def test_retried_agentic_request_that_recovers_bills_the_failed_attempts_first_call_on_success( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter -) -> None: - outcome, shared_logging, first_calls, hook = await _route_agentic_request_with_retries( - monkeypatch, respx_mock, ("litellm.RateLimitError", "42") - ) - await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10) +async def _successful_agentic_request_cost_report( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...] +) -> tuple[int, Mapping[str, object], Mapping[str, str | None]]: + from litellm.proxy.common_request_processing import cost_headers + outcome, shared_logging, _, hook = await _route_agentic_request_with_retries(monkeypatch, respx_mock, follow_ups) + await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10) assert isinstance(outcome, litellm.ModelResponse) success: Final = shared_logging.model_call_details["standard_logging_object"] - follow_up_cost: Final = sum( - litellm.cost_per_token( - model="gpt-4o-mini", - prompt_tokens=outcome.usage.prompt_tokens, - completion_tokens=outcome.usage.completion_tokens, - ) + billed: Final = { + field: success[field] for field in ("response_cost", "prompt_tokens", "completion_tokens", "cost_breakdown") + } + return hook.plans_built, billed, cost_headers(success["response_cost"], shared_logging) + + +@pytest.mark.asyncio +async def test_retried_agentic_request_that_recovers_reports_the_cost_of_a_first_try_success( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + first_try_attempts, first_try_billed, first_try_headers = await _successful_agentic_request_cost_report( + monkeypatch, respx_mock, ("42",) ) - failed_first_call_cost: Final = sum( - litellm.cost_per_token(model="gpt-4o-mini", prompt_tokens=50, completion_tokens=7) + recovered_attempts, recovered_billed, recovered_headers = await _successful_agentic_request_cost_report( + monkeypatch, respx_mock, ("litellm.RateLimitError", "42") ) - assert first_calls.call_count == 2 - assert success["prompt_tokens"] == outcome.usage.prompt_tokens + _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"] - assert ( - success["completion_tokens"] - == outcome.usage.completion_tokens + _AGENTIC_FIRST_CALL_USAGE["completion_tokens"] - ) - assert success["response_cost"] == pytest.approx(follow_up_cost + failed_first_call_cost) + + assert (first_try_attempts, recovered_attempts) == (1, 2) + assert recovered_billed == first_try_billed + assert recovered_headers == first_try_headers def _responses_sse(*events: Mapping[str, object]) -> str: From 7fd14e5beb43aaef97bd192b308a4b972d8cc8a2 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 00:32:12 -0700 Subject: [PATCH 08/12] fix(agentic): stop retrying a request whose agentic follow-up failed and bill every round it served An error raised by an agentic follow-up after the first call was served now carries a typed mark, so the router's retry loops, the mid-stream retry, and litellm's num_retries wrapper raise it as is instead of rerunning the whole request. A failure of the first call itself still retries, and fallbacks still run. The failure row bills every round the loop served before the failure, at any depth, and a later attempt that succeeds bills what the failed attempts paid on its success row, with the cost breakdown and the response cost headers adding up. The error response's x-litellm-response-cost reports the same amount its failure row bills. Failure rows outside agentic hooks keep the prompt-token estimate they had before. A refused tool call now matches by id among the refused ids, or by name among refused calls that carry no id, so a name-only refusal next to an id refusal is no longer kept. --- .../agentic_followup_failure.py | 27 ++ .../agentic_loop_refusal.py | 14 +- .../chat_completion_agentic_loop.py | 12 +- litellm/litellm_core_utils/litellm_logging.py | 93 +++-- litellm/proxy/common_request_processing.py | 7 +- .../proxy/hooks/proxy_track_cost_callback.py | 40 ++- litellm/proxy/utils.py | 4 +- litellm/router.py | 9 +- litellm/utils.py | 15 +- .../test_chat_completion_agentic_loop.py | 144 +++++++- .../custom_httpx/test_llm_http_handler.py | 50 +-- tests/unit/llms/openai/test_openai.py | 50 +-- .../proxy/test_common_request_processing.py | 56 +++ ..._utils_model_creation_and_error_logging.py | 64 ++-- .../mcp/test_aresponses_api_with_mcp.py | 47 ++- tests/unit/test_router/test_router.py | 322 ++++++++++++------ 16 files changed, 711 insertions(+), 243 deletions(-) create mode 100644 litellm/litellm_core_utils/agentic_followup_failure.py diff --git a/litellm/litellm_core_utils/agentic_followup_failure.py b/litellm/litellm_core_utils/agentic_followup_failure.py new file mode 100644 index 00000000000..1847d0f0cd7 --- /dev/null +++ b/litellm/litellm_core_utils/agentic_followup_failure.py @@ -0,0 +1,27 @@ +import weakref +from typing import Final, TypeAlias + +from litellm.types.utils import Usage + +ServedRound: TypeAlias = tuple[Usage, float] + +_followup_failures: Final[weakref.WeakKeyDictionary[BaseException, tuple[ServedRound, ...]]] = ( + weakref.WeakKeyDictionary() +) + + +def _can_be_marked(error: BaseException) -> bool: + return type(error).__weakrefoffset__ != 0 + + +def mark_agentic_followup_failure(error: BaseException, served_rounds: tuple[ServedRound, ...]) -> None: + if _can_be_marked(error): + _followup_failures[error] = served_rounds + + +def is_agentic_followup_failure(error: BaseException) -> bool: + return _can_be_marked(error) and error in _followup_failures + + +def rounds_served_before(error: BaseException) -> tuple[ServedRound, ...]: + return _followup_failures.get(error, ()) if _can_be_marked(error) else () diff --git a/litellm/litellm_core_utils/agentic_loop_refusal.py b/litellm/litellm_core_utils/agentic_loop_refusal.py index abd31e856f6..f3ee15dffdd 100644 --- a/litellm/litellm_core_utils/agentic_loop_refusal.py +++ b/litellm/litellm_core_utils/agentic_loop_refusal.py @@ -40,19 +40,19 @@ async def every_claimed_tool_call( return MappingProxyType({"tool_calls": (*_claimed_calls(refused_tool_calls), *chain.from_iterable(claims))}) +def _call_ids(call: object) -> frozenset[str]: + return _strings((message_field(call, "id"), message_field(call, "call_id"))) + + def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]: calls: Final = _claimed_calls(tool_calls) - ids: Final = _strings( - chain.from_iterable((message_field(call, "id"), message_field(call, "call_id")) for call in calls) - ) - names: Final = _strings(message_field(call, "name") for call in calls) + ids: Final = frozenset(chain.from_iterable(_call_ids(call) for call in calls)) + names: Final = _strings(message_field(call, "name") for call in calls if not _call_ids(call)) return ids, names def is_refused_call(call_id: object, name: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool: - if isinstance(call_id, str) and refused_ids: - return call_id in refused_ids - return name in refused_names + return (isinstance(call_id, str) and call_id in refused_ids) or name in refused_names def is_refused_item( diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 3021d417b9e..7de27f104f6 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -10,6 +10,7 @@ from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.agentic_followup_failure import mark_agentic_followup_failure, rounds_served_before from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs from litellm.litellm_core_utils.agentic_loop_refusal import every_claimed_tool_call, finalize_refused_chat_turn from litellm.litellm_core_utils.agentic_loop_settings import ( @@ -54,9 +55,14 @@ async def bill_first_round_on_failure( ) -> _DispatchResult: try: return await dispatch - except Exception: - if not stream and isinstance(logging_obj, LiteLLMLoggingObject): - logging_obj.record_earlier_round_for_failure(response) + except Exception as error: + if stream or not isinstance(logging_obj, LiteLLMLoggingObject): + mark_agentic_followup_failure(error, rounds_served_before(error)) + raise + this_round: Final = logging_obj.served_round(response) + served_rounds: Final = (*rounds_served_before(error), *(() if this_round is None else (this_round,))) + mark_agentic_followup_failure(error, served_rounds) + logging_obj.record_served_rounds_for_failure(served_rounds) raise diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index a1257340d59..edb7856fc55 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -50,6 +50,7 @@ from litellm.constants import ( REDACTED_BY_LITELLM, ) from litellm.cost_calculator import ( + BaseTokenUsageProcessor, RealtimeAPITokenUsageProcessor, ResponsesWebSocketTokenUsageProcessor, get_usage_object, @@ -70,6 +71,7 @@ from litellm.integrations.deepeval.deepeval import DeepEvalLogger from litellm.integrations.langtrace import langtrace_trace_endpoint from litellm.integrations.mlflow import MlflowLogger from litellm.integrations.sqs import SQSLogger +from litellm.litellm_core_utils.agentic_followup_failure import ServedRound from litellm.litellm_core_utils.classifier_logging import ( classifier_audit_fields, classifier_input_snapshot, @@ -571,6 +573,7 @@ _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata" _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object]) _SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object]) _REDACTED_RESULT: Final = TypeAdapter(object) +_FAILED_ATTEMPTS_USAGE: Final = TypeAdapter(tuple[ServedRound, ...]) _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information" @@ -2636,35 +2639,43 @@ class Logging(LiteLLMLoggingBaseClass): own usage while the request's failure log still bills every attempt.""" usage: Final = self._model_call_details_view.get("combined_usage_object") response_cost: Final = self._model_call_details_view.get("response_cost") + billed_by_continuation: Final = self.model_call_details.pop("stash_billed_by_continuation", False) is True self.model_call_details.pop("combined_usage_object", None) self.model_call_details.pop("response_cost", None) - if isinstance(usage, Usage): - self.model_call_details["current_attempt_api_call_start_time"] = None - self.model_call_details["failed_attempts_usage"] = ( - *self.model_call_details.get("failed_attempts_usage", ()), - (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0), + if not isinstance(usage, Usage): + return + attempt: Final = (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0) + self.model_call_details["current_attempt_api_call_start_time"] = None + self.model_call_details["failed_attempts_usage"] = ( + *self.model_call_details.get("failed_attempts_usage", ()), + attempt, + ) + if not billed_by_continuation: + self.model_call_details["failed_attempts_owed_on_success"] = ( + *self.model_call_details.get("failed_attempts_owed_on_success", ()), + attempt, ) + def mark_stash_billed_by_continuation(self) -> None: + self.model_call_details["stash_billed_by_continuation"] = True + def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None: """Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw.""" self.model_call_details["continued_partial_usage"] = usage self.model_call_details["continued_partial_cost"] = response_cost self.model_call_details["continued_response_id"] = response_id - def _add_continued_partial_spend(self, response: object) -> None: + def _add_continued_partial_spend(self, response: object) -> Usage | None: partial_usage: Final = self.model_call_details.get("continued_partial_usage") partial_cost: Final = self.model_call_details.get("continued_partial_cost") own_usage: Final = getattr(response, "usage", None) own_cost: Final = self.model_call_details.get("response_cost") if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)): - return - from litellm.cost_calculator import BaseTokenUsageProcessor - - self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects( - usage_objects=[own_usage, partial_usage] - ) + return own_usage if isinstance(own_usage, Usage) else None + combined_usage: Final = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[own_usage, partial_usage]) + self.model_call_details["combined_usage_object"] = combined_usage if not isinstance(own_cost, (int, float)): - return + return combined_usage self.model_call_details["response_cost"] = own_cost + partial_cost if self.cost_breakdown is not None: self.cost_breakdown = self.cost_breakdown | CostBreakdown( @@ -2674,6 +2685,35 @@ class Logging(LiteLLMLoggingBaseClass): "mid_stream_partial_cost": partial_cost, }, ) + return combined_usage + + def _add_failed_attempts_spend(self, own_usage: Usage | None) -> None: + own_cost: Final = self.model_call_details.get("response_cost") + if own_usage is None or not isinstance(own_cost, (int, float)): + return + failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python( + ( + *self.model_call_details.get("failed_attempts_billed_on_success", ()), + *self.model_call_details.pop("failed_attempts_owed_on_success", ()), + ) + ) + if not failed_attempts: + return + failed_attempts_cost: Final = sum(cost for _, cost in failed_attempts) + self.model_call_details.pop("failed_attempts_usage", None) + self.model_call_details["failed_attempts_billed_on_success"] = failed_attempts + self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects( + usage_objects=[own_usage, *(usage for usage, _ in failed_attempts)] + ) + self.model_call_details["response_cost"] = own_cost + failed_attempts_cost + if self.cost_breakdown is not None: + self.cost_breakdown = self.cost_breakdown | CostBreakdown( + total_cost=own_cost + failed_attempts_cost, + additional_costs={ + **self.cost_breakdown.get("additional_costs", {}), + "failed_attempts_cost": failed_attempts_cost, + }, + ) def claim_stream_failure_report(self) -> bool: if self._stream_failure_claimed: @@ -2681,13 +2721,22 @@ class Logging(LiteLLMLoggingBaseClass): self._stream_failure_claimed = True return True - def record_earlier_round_for_failure(self, response: object) -> None: + def served_round(self, response: object) -> ServedRound | None: usage: Final = get_usage_object(completion_response=response) - if usage is not None: - earlier_round_price: Final = self._price_without_recording( - response, cache_hit=None, ignore_stamped_cost=False - ) - self.record_partial_usage_for_failure(usage, earlier_round_price.cost or 0.0) + if usage is None: + return None + price: Final = self._price_without_recording(response, cache_hit=None, ignore_stamped_cost=False) + return usage, price.cost or 0.0 + + def record_served_rounds_for_failure(self, served_rounds: tuple[ServedRound, ...]) -> None: + if not served_rounds: + return + self.record_partial_usage_for_failure( + served_rounds[0][0] + if len(served_rounds) == 1 + else BaseTokenUsageProcessor.combine_usage_objects(usage_objects=[usage for usage, _ in served_rounds]), + sum(cost for _, cost in served_rounds), + ) def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None: """Bill a fully streamed response on the failure log when a post-call hook rejects it.""" @@ -2935,6 +2984,7 @@ class Logging(LiteLLMLoggingBaseClass): elif "response_cost" in hidden_params: self.model_call_details["response_cost"] = hidden_params["response_cost"] self._record_zero_cost_diagnostic(logging_result, hidden_params["response_cost"]) + self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result)) elif (existing_cost := self.model_call_details.get("response_cost")) is not None and ( existing_cost != 0 or self.call_type == CallTypes.pass_through.value # pyright: ignore[reportUnknownMemberType] # Logging.call_type is untyped ): @@ -2944,6 +2994,7 @@ class Logging(LiteLLMLoggingBaseClass): pass else: self.model_call_details["response_cost"] = self.response_cost_calculator(result=logging_result) + self._add_failed_attempts_spend(get_usage_object(completion_response=logging_result)) if not build_logging_payload: return None @@ -3449,7 +3500,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["response_cost"] = self.response_cost_calculator( result=complete_streaming_response ) - self._add_continued_partial_spend(complete_streaming_response) + self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response)) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) logged_streaming_response: Final = self._with_continued_response_id( self._with_client_facing_stream_model(complete_streaming_response) @@ -4041,7 +4092,7 @@ class Logging(LiteLLMLoggingBaseClass): self.model, ) self.model_call_details["response_cost"] = None - self._add_continued_partial_spend(complete_streaming_response) + self._add_failed_attempts_spend(self._add_continued_partial_spend(complete_streaming_response)) self._merge_hidden_params_from_response_into_metadata(complete_streaming_response) diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 762f552dadc..27993d439cd 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -53,6 +53,7 @@ from litellm.constants import ( ) from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.otel.runtime import phase_event +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.bug_report import ( allowlisted, bug_report_notice, @@ -129,6 +130,7 @@ from litellm.proxy.common_utils.sse_keepalive import ( from litellm.proxy.dd_span_tagger import DDSpanTagger from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash +from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost from litellm.proxy.native_compaction import with_proxy_compaction_executor from litellm.proxy.route_llm_request import ( endpoint_unsupported_by_model, @@ -4342,6 +4344,7 @@ class ProxyBaseLLMRequestProcessing: """Raises ProxyException (OpenAI API compatible) if an exception is raised""" log_llm_api_exception(e, self.litellm_call_id) billing_logging_obj: Final = _REQUEST_FIELDS.validate_python(self.data).get("litellm_logging_obj") + failed_in_agentic_follow_up: Final = is_agentic_followup_failure(e) # Allow callbacks to transform the error response transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, @@ -4378,7 +4381,9 @@ class ProxyBaseLLMRequestProcessing: model_id=model_id, version=version, response_cost=( - _litellm_logging_obj.finalized_cost.amount + failure_row_response_cost(self.data) + if failed_in_agentic_follow_up + else _litellm_logging_obj.finalized_cost.amount if isinstance(_litellm_logging_obj, LiteLLMLoggingObj) and _litellm_logging_obj.finalized_cost is not None else 0 diff --git a/litellm/proxy/hooks/proxy_track_cost_callback.py b/litellm/proxy/hooks/proxy_track_cost_callback.py index 0d333d4fd08..5781170a47d 100644 --- a/litellm/proxy/hooks/proxy_track_cost_callback.py +++ b/litellm/proxy/hooks/proxy_track_cost_callback.py @@ -218,6 +218,32 @@ def _guardrail_info_lifted_from_litellm_metadata( return {"standard_logging_guardrail_information": guardrail_info} +def _failure_row_guardrail_information(request_data: Mapping[str, object]) -> object: + metadata: Final = _str_keyed_fields(request_data.get("metadata")) + lifted: Final = _guardrail_info_lifted_from_litellm_metadata( + _str_keyed_fields(request_data.get("litellm_metadata")), + guardrail_info_already_set="standard_logging_guardrail_information" in metadata, + ) + return {**metadata, **lifted}.get("standard_logging_guardrail_information") + + +def failure_row_response_cost(request_data: Mapping[str, object]) -> float: + stashed_cost: Final = request_data.get("response_cost") + recovered_stream_cost: Final = ( + max(float(stashed_cost), 0.0) + if isinstance(request_data.get("combined_usage_object"), litellm.Usage) + and isinstance(stashed_cost, (int, float)) + else 0.0 + ) + finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object")) + finalized_amount: Final = finalized_failure.get("response_cost") + return ( + float(finalized_amount) + if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float)) + else recovered_stream_cost + guardrail_information_cost(_failure_row_guardrail_information(request_data)) + ) + + def _redaction_opt_in_lifted_from_litellm_metadata( litellm_metadata: ObjectMapping, *, caller_set_headers: bool ) -> ObjectMapping: @@ -543,19 +569,7 @@ class _ProxyDBLogger(CustomLogger): # recovered cost onto request_data (the usage rides along in # ``combined_usage_object`` for the token columns), so attribute the # real partial spend to this failure row instead of zero. - recovered_stream_cost: Final = ( - max(float(request_data.get("response_cost") or 0.0), 0.0) - if isinstance(request_data.get("combined_usage_object"), litellm.Usage) - else 0.0 - ) - finalized_failure: Final = _str_keyed_fields(request_data.get("standard_logging_object")) - finalized_amount: Final = finalized_failure.get("response_cost") - recovered_response_cost: Final = ( - float(finalized_amount) - if finalized_failure.get("cost_finalized") is True and isinstance(finalized_amount, (int, float)) - else recovered_stream_cost - + guardrail_information_cost(existing_metadata.get("standard_logging_guardrail_information")) - ) + recovered_response_cost: Final = failure_row_response_cost(request_data) failure_snapshot_with_late_guardrails: Final = _failure_snapshot_with_guardrails_recorded_after_it( request_data.get("standard_logging_object"), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 3e905363102..ddaa8d2e69d 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1359,16 +1359,16 @@ def _failure_usage_to_lift( base_model: str | None = None, dispatched_call_type: str | None = None, ) -> tuple[object, object] | None: - failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ())) current_attempt: Final = _current_failure_usage_to_lift( model_call_details, request_body, - dispatched and (provider_billed or not failed_attempts), + dispatched, provider_billed, deployment_id, base_model, dispatched_call_type=dispatched_call_type, ) + failed_attempts: Final = _FAILED_ATTEMPTS_USAGE.validate_python(model_call_details.get("failed_attempts_usage", ())) if not failed_attempts: return current_attempt current_usage, current_cost = current_attempt if current_attempt is not None else (None, None) diff --git a/litellm/router.py b/litellm/router.py index 14879ae78b5..6c3777ee40d 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -106,6 +106,7 @@ from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_ from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.otel.routing import routing_decision_attributes from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.batch_utils import ( batch_create_routing, @@ -3902,6 +3903,8 @@ class Router(DeprecatedRouterMembers): if continuation_output is not None else None ) + if cut_partial is not None: + model_response.logging_obj.mark_stash_billed_by_continuation() if continue_after_content: emitted_tokens: Final = int( getattr(complete_response_object_usage, "completion_tokens", 0) or 0 @@ -7123,6 +7126,8 @@ class Router(DeprecatedRouterMembers): last_error = e # rebind-ok: the newest failure is what the fallback chain and the caller see for attempt in range(attempted_retries_for_request(initial_kwargs), budget): trigger = _mid_stream_retry_trigger(last_error) + if is_agentic_followup_failure(trigger): + return _StreamRetriesExhausted(last_error) if not policy_applies and not self._stream_should_retry( trigger, healthy_deployments, all_deployments, initial_kwargs ): @@ -9339,7 +9344,7 @@ class Router(DeprecatedRouterMembers): response = add_retry_headers_to_response(response=response, attempted_retries=0, max_retries=None) return response except Exception as e: - if is_guardrail_intervention(e): + if is_guardrail_intervention(e) or is_agentic_followup_failure(e): raise current_attempt = None original_exception = e @@ -9446,6 +9451,8 @@ class Router(DeprecatedRouterMembers): return response except Exception as e: + if is_agentic_followup_failure(e): + raise # Always track the latest error so we raise the most # recent exception instead of the first one. original_exception = e diff --git a/litellm/utils.py b/litellm/utils.py index 2ca8a49f968..088a5a5f7ce 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -83,6 +83,7 @@ from litellm.constants import ( PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO, TOOL_CHOICE_OBJECT_TOKEN_COUNT, ) +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.core_helpers import ( bind_budget_reservation_to_callbacks, normalize_drop_params, @@ -2235,7 +2236,7 @@ def client(original_function): is_completion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( - num_retries and not is_completion_litellm_router_call + num_retries and not is_completion_litellm_router_call and not is_agentic_followup_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying if ( isinstance(e, openai.APIError) @@ -2269,7 +2270,7 @@ def client(original_function): is_responses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=False) if ( - num_retries and not is_responses_litellm_router_call + num_retries and not is_responses_litellm_router_call and not is_agentic_followup_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying if ( isinstance(e, openai.APIError) @@ -2554,7 +2555,10 @@ def client(original_function): is_acompletion_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) if ( - num_retries and not is_acompletion_litellm_router_call and not isinstance(e, ImportError) + num_retries + and not is_acompletion_litellm_router_call + and not isinstance(e, ImportError) + and not is_agentic_followup_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: litellm.num_retries = None # set retries to None to prevent infinite loops @@ -2585,7 +2589,10 @@ def client(original_function): is_aresponses_litellm_router_call: Final = _is_litellm_router_call(kwargs, is_async=True) if ( - num_retries and not is_aresponses_litellm_router_call and not isinstance(e, ImportError) + num_retries + and not is_aresponses_litellm_router_call + and not isinstance(e, ImportError) + and not is_agentic_followup_failure(e) ): # only enter this if call is not from litellm router/proxy. router has it's own logic for retrying try: litellm.num_retries = None # set retries to None to prevent infinite loops diff --git a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py index ac3707adf13..0a5afc1ecd2 100644 --- a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -22,6 +22,7 @@ they stay out of the body even without it. import datetime import json +from collections.abc import Mapping, Sequence from typing import Any, Dict, Final, List, Optional, Tuple from unittest.mock import AsyncMock, MagicMock, patch @@ -264,8 +265,6 @@ class _GateOnlyLogger(CustomLogger): class _RecordingGateLogger(_GateOnlyLogger): - """Also keeps every follow-up response the loop hands to the post hook.""" - def __init__(self, plan: AgenticLoopPlan, tool_calls: Dict[str, Any]) -> None: super().__init__(plan=plan, tool_calls=tool_calls) self.followups: tuple[ModelResponse, ...] = () @@ -622,6 +621,35 @@ async def test_refused_rerun_drops_every_hooks_internal_call_and_keeps_the_clien assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"] +@pytest.mark.asyncio +async def test_refused_rerun_drops_a_call_claimed_only_by_name_next_to_one_claimed_by_id(restore_callbacks): + response: Final = ModelResponse( + choices=[ + Choices( + finish_reason="tool_calls", + message=Message( + role="assistant", + content=None, + tool_calls=[ + _tool_call("call_abc", "litellm_code_execution"), + _tool_call("call_search", "litellm_web_search"), + _tool_call("call_client", "lookup"), + ], + ), + ) + ] + ) + + result: Final = await _run_refused_rerun( + response, + _REFUSING_RAILS[0], + gate_tool_calls={"tool_calls": [{"id": "call_abc"}]}, + other_hooks_tool_calls=({"tool_calls": [{"name": "litellm_web_search"}]},), + ) + + assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"] + + @pytest.mark.asyncio @pytest.mark.parametrize( "response, gate_tool_calls", @@ -716,6 +744,118 @@ async def test_acompletion_bills_the_first_call_on_the_failure_log_when_the_foll ) +def _tool_call_turn(call_id: str, usage: Usage) -> ModelResponse: + return ModelResponse( + model="gpt-4o-mini", + choices=[ + Choices( + finish_reason="tool_calls", + message=Message(role="assistant", content=None, tool_calls=[_tool_call(call_id, "litellm_web_search")]), + ) + ], + usage=usage, + ) + + +class _TwoRoundLogger(CustomLogger): + def __init__(self, middle_round: ModelResponse) -> None: + super().__init__() + self._follow_ups: Final = (middle_round, "litellm.RateLimitError") + self.logging_objs: tuple[object, ...] = () + + async def async_should_run_agentic_loop( + self, + response: object, + model: str, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, + stream: bool, + custom_llm_provider: str, + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: + assert isinstance(response, ModelResponse) + calls: Final = response.choices[0].message.tool_calls or [] + return bool(calls), {"tool_calls": [{"id": call.id} for call in calls]} + + async def async_build_agentic_loop_plan( + self, + tools: Mapping[str, object], + model: str, + messages: Sequence[Mapping[str, object]], + response: object, + anthropic_messages_provider_config: object, + anthropic_messages_optional_request_params: Mapping[str, object], + logging_obj: object, + stream: bool, + kwargs: Mapping[str, object], + ) -> AgenticLoopPlan: + follow_up: Final = self._follow_ups[len(self.logging_objs)] + self.logging_objs = (*self.logging_objs, logging_obj) + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch(messages=_patched_messages(), kwargs={"mock_response": follow_up}), + ) + + +@pytest.mark.asyncio +async def test_acompletion_bills_every_served_round_on_the_failure_log_when_a_later_follow_up_fails( + restore_callbacks, monkeypatch: pytest.MonkeyPatch +): + first_usage: Final = Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8) + middle_usage: Final = Usage(prompt_tokens=11, completion_tokens=4, total_tokens=15) + logger: Final = _TwoRoundLogger(middle_round=_tool_call_turn("call_next", middle_usage)) + monkeypatch.setattr(litellm, "callbacks", [logger]) + mock_client: Final = MagicMock() + mock_client.post = AsyncMock(return_value=_raw_response_for(_tool_call_turn("call_abc", first_usage))) + + with pytest.raises(litellm.RateLimitError): + await litellm.acompletion( + model="openai/gpt-4o-mini", + messages=[{"role": "user", "content": "what is 6*7?"}], + api_key="sk-test", + client=mock_client, + num_retries=0, + ) + + first_logging_obj, _ = logger.logging_objs + assert isinstance(first_logging_obj, Logging) + price: Final = litellm.model_cost["gpt-4o-mini"] + billed_usage: Final = first_logging_obj.model_call_details["combined_usage_object"] + assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == (5 + 11, 3 + 4) + assert first_logging_obj.model_call_details["response_cost"] == pytest.approx( + (5 + 11) * price["input_cost_per_token"] + (3 + 4) * price["output_cost_per_token"] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +async def test_a_follow_up_failing_with_a_builtin_error_reaches_the_caller_unchanged(stream: bool): + from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure + + sandbox_error: Final = RuntimeError("sandbox crashed") + + async def failing_follow_up() -> ModelResponse: + raise sandbox_error + + with pytest.raises(RuntimeError) as raised: + await bill_first_round_on_failure( + failing_follow_up(), + logging_obj=Logging( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "hi"}], + stream=stream, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-builtin-error", + function_id="fn-builtin-error", + ), + response=_tool_call_turn("call_abc", Usage(prompt_tokens=5, completion_tokens=3, total_tokens=8)), + stream=stream, + ) + + assert raised.value is sandbox_error + + class _RunOnlyLoopLogger(CustomLogger): def __init__(self, result: ModelResponse) -> None: super().__init__() diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 3d6069a3e80..10ae5ed639e 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -5,7 +5,7 @@ import json import logging import threading import time -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from types import MappingProxyType from typing import Final from unittest.mock import AsyncMock, Mock, patch @@ -5369,12 +5369,12 @@ class _PlannedChatLoop(CustomLogger): self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: claimed: Final = [ {"id": call.id, "name": call.function.name} for call in _tool_calls_of(response) @@ -5384,14 +5384,14 @@ class _PlannedChatLoop(CustomLogger): async def async_build_chat_completion_agentic_loop_plan( self, - tools: dict, + tools: Mapping[str, object], model: str, - messages: list[dict], + messages: Sequence[Mapping[str, object]], response: object, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: Mapping[str, object], ) -> AgenticLoopPlan: self.logging_objs.append(logging_obj) return self._plan @@ -5406,24 +5406,24 @@ class _RunOnlyChatLoop(CustomLogger): self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: return True, {"tool_calls": [{"id": "call_search", "name": "litellm_web_search"}]} async def async_run_chat_completion_agentic_loop( self, - tools: dict, + tools: Mapping[str, object], model: str, - messages: list[dict], + messages: Sequence[Mapping[str, object]], response: object, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: Mapping[str, object], ) -> object: return self._result @@ -5599,12 +5599,12 @@ class _SearchAgainLoop(CustomLogger): self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: searches: Final = [ {"id": call.id, "name": call.function.name} for call in response.choices[0].message.tool_calls or [] @@ -5614,14 +5614,14 @@ class _SearchAgainLoop(CustomLogger): async def async_build_chat_completion_agentic_loop_plan( self, - tools: dict, + tools: Mapping[str, object], model: str, - messages: list[dict], + messages: Sequence[Mapping[str, object]], response: object, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: Mapping[str, object], ) -> AgenticLoopPlan: return AgenticLoopPlan( run_agentic_loop=True, diff --git a/tests/unit/llms/openai/test_openai.py b/tests/unit/llms/openai/test_openai.py index 0cb955dd356..0adfe179f01 100644 --- a/tests/unit/llms/openai/test_openai.py +++ b/tests/unit/llms/openai/test_openai.py @@ -3,7 +3,7 @@ import gzip import importlib import json import os -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable, Mapping, Sequence from types import ModuleType from typing import Final, Literal, Protocol from unittest.mock import MagicMock, Mock, patch @@ -236,12 +236,12 @@ class _SearchingChatGate(CustomLogger): self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: return True, {"tool_calls": [{"id": "call-1"}]} @@ -253,14 +253,14 @@ class _FailingFollowUpChatLoop(_SearchingChatGate): async def async_run_chat_completion_agentic_loop( self, - tools: dict, + tools: Mapping[str, object], model: str, - messages: list[dict], + messages: Sequence[Mapping[str, object]], response: object, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: Mapping[str, object], ) -> object: self.runs += 1 self.logging_objs.append(logging_obj) @@ -272,26 +272,26 @@ class _RaisingAgenticGate(CustomLogger): self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: raise ValueError("gate bug") class _AnsweringChatLoop(_SearchingChatGate): async def async_run_chat_completion_agentic_loop( self, - tools: dict, + tools: Mapping[str, object], model: str, - messages: list[dict], + messages: Sequence[Mapping[str, object]], response: object, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: Mapping[str, object], ) -> object: self.runs += 1 return litellm.ModelResponse( @@ -490,12 +490,12 @@ class _SearchAgainLoop(CustomLogger): self, response: object, model: str, - messages: list[dict], - tools: list[dict] | None, + messages: Sequence[Mapping[str, object]], + tools: Sequence[Mapping[str, object]] | None, stream: bool, custom_llm_provider: str, - kwargs: dict, - ) -> tuple[bool, dict]: + kwargs: Mapping[str, object], + ) -> tuple[bool, dict[str, object]]: searches: Final = [ {"id": call.id, "name": call.function.name} for call in response.choices[0].message.tool_calls or [] @@ -505,14 +505,14 @@ class _SearchAgainLoop(CustomLogger): async def async_build_chat_completion_agentic_loop_plan( self, - tools: dict, + tools: Mapping[str, object], model: str, - messages: list[dict], + messages: Sequence[Mapping[str, object]], response: object, - optional_params: dict, + optional_params: Mapping[str, object], logging_obj: object, stream: bool, - kwargs: dict, + kwargs: Mapping[str, object], ) -> AgenticLoopPlan: return AgenticLoopPlan( run_agentic_loop=True, diff --git a/tests/unit/proxy/test_common_request_processing.py b/tests/unit/proxy/test_common_request_processing.py index a7596b51e3e..bdeea581c1c 100644 --- a/tests/unit/proxy/test_common_request_processing.py +++ b/tests/unit/proxy/test_common_request_processing.py @@ -4292,6 +4292,62 @@ class TestHandleLLMApiExceptionPricesWhatTheServerDispatched: assert resent >= dispatched_after_trim +class TestHandleLLMApiExceptionReportsWhatAFailedAgenticFollowUpBills: + async def _error_response_cost(self, raise_the_follow_up_error: bool) -> float: + from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure + from litellm.types.utils import ModelResponse, Usage + + processing_obj: Final = ProxyBaseLLMRequestProcessing( + data={"model": "gpt-4o", "messages": [{"role": "user", "content": "what is 6*7?"}]} + ) + caller: Final = ProxyUserAPIKeyAuth(request_route="/v1/chat/completions") + mock_request: Final = MagicMock(spec=Request) + mock_request.headers = {} + proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging.alert_types = [] + _, logging_obj = await processing_obj.common_processing_pre_call_logic( + request=mock_request, + general_settings={}, + user_api_key_dict=caller, + proxy_logging_obj=proxy_logging, + proxy_config=MagicMock(spec=ProxyConfig), + route_type="acompletion", + ) + logging_obj.model_call_details.update(first_api_call_start_time=datetime.datetime.now()) + follow_up_error: Final = litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o") + + async def failing_follow_up() -> ModelResponse: + raise follow_up_error + + with pytest.raises(litellm.RateLimitError): + await bill_first_round_on_failure( + failing_follow_up(), + logging_obj=logging_obj, + response=ModelResponse( + model="gpt-4o", usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672) + ), + stream=False, + ) + with pytest.raises(ProxyException) as raised: + await processing_obj.handle_llm_api_exception( + e=follow_up_error + if raise_the_follow_up_error + else litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o"), + user_api_key_dict=caller, + proxy_logging_obj=proxy_logging, + ) + return float(raised.value.headers["x-litellm-response-cost"]) + + @pytest.mark.asyncio + async def test_the_error_response_reports_the_served_first_call_its_failure_row_bills(self) -> None: + price: Final = litellm.model_cost["gpt-4o"] + + assert await self._error_response_cost(raise_the_follow_up_error=True) == pytest.approx( + 559 * price["input_cost_per_token"] + 113 * price["output_cost_per_token"] + ) + assert await self._error_response_cost(raise_the_follow_up_error=False) == 0 + + @pytest.mark.asyncio class TestHandleLLMApiExceptionDictDetail: """ diff --git a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py index 291231a4aaf..293116d575f 100644 --- a/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py +++ b/tests/unit/proxy/test_proxy_utils_model_creation_and_error_logging.py @@ -3561,13 +3561,15 @@ def test_an_earlier_timed_out_responses_attempt_counts_its_input_from_the_newest [None, 1700000001.0], ids=["retry-not-dispatched", "retry-dispatched"], ) -def test_an_unbilled_retry_failure_adds_no_estimated_tokens_to_the_billed_attempts( +def test_a_retry_failure_bills_input_only_for_dispatched_attempts( current_attempt_api_call_start_time: float | None, ) -> None: - from litellm.proxy.utils import _failure_fields_to_lift + from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift from litellm.types.utils import Usage prior_usage: Final = Usage(prompt_tokens=4, completion_tokens=2, total_tokens=6) + estimated_retry_usage: Final = _estimate_dispatched_failure_usage(FAILURE_USAGE_MODEL, ONE_USER_MESSAGE, None) + assert estimated_retry_usage is not None lifted: Final = _failure_fields_to_lift( { @@ -3586,43 +3588,63 @@ def test_an_unbilled_retry_failure_adds_no_estimated_tokens_to_the_billed_attemp usage: Final = lifted["combined_usage_object"] assert isinstance(usage, Usage) - assert (usage.prompt_tokens, usage.completion_tokens, usage.total_tokens) == (4, 2, 6) + expected_retry_tokens: Final = ( + estimated_retry_usage.prompt_tokens if current_attempt_api_call_start_time is not None else 0 + ) + assert usage.prompt_tokens == prior_usage.prompt_tokens + expected_retry_tokens + assert usage.completion_tokens == prior_usage.completion_tokens + assert usage.total_tokens == prior_usage.total_tokens + expected_retry_tokens assert lifted["response_cost"] == 0.25 -def test_a_responses_failure_after_a_retried_follow_up_bills_only_the_first_calls_usage() -> None: - from litellm.litellm_core_utils.litellm_logging import Logging +@pytest.mark.asyncio +async def test_an_agentic_follow_up_failure_row_bills_each_served_first_call_without_an_estimate() -> None: + import litellm + from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure from litellm.proxy.utils import _failure_fields_to_lift from litellm.types.utils import ModelResponse, Usage - responses_input: Final = [{"role": "user", "content": "Search the web for the latest LiteLLM release"}] logging_obj: Final = Logging( model=FAILURE_USAGE_MODEL, - messages=responses_input, + messages=ONE_USER_MESSAGE, stream=False, - call_type="aresponses", + call_type="acompletion", start_time=real_datetime.datetime.now(), - litellm_call_id="call-responses-retry", - function_id="fn-responses-retry", + litellm_call_id="call-agentic-follow-up", + function_id="fn-agentic-follow-up", ) logging_obj.update_environment_variables( model=FAILURE_USAGE_MODEL, user="", optional_params={}, litellm_params={"custom_llm_provider": "openai"} ) - logging_obj.pre_call(input=responses_input, api_key="sk-test") - first_call: Final = ModelResponse(usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672)) - logging_obj.record_earlier_round_for_failure(first_call) - first_call_cost: Final = logging_obj.model_call_details["response_cost"] - logging_obj.carry_failed_attempt_usage() - logging_obj.pre_call(input=responses_input, api_key="sk-test") - lifted: Final = _failure_fields_to_lift( - {"litellm_logging_obj": logging_obj}, provider_billed=False, dispatched_call_type="aresponses" + async def failing_follow_up() -> ModelResponse: + raise litellm.RateLimitError(message="rate limited", llm_provider="openai", model=FAILURE_USAGE_MODEL) + + async def serve_first_call_then_fail_its_follow_up(usage: Usage) -> float: + logging_obj.pre_call(input=ONE_USER_MESSAGE, api_key="sk-test") + with pytest.raises(litellm.RateLimitError): + await bill_first_round_on_failure( + failing_follow_up(), + logging_obj=logging_obj, + response=ModelResponse(model=FAILURE_USAGE_MODEL, usage=usage), + stream=False, + ) + return logging_obj.model_call_details["response_cost"] + + primary_cost: Final = await serve_first_call_then_fail_its_follow_up( + Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672) + ) + logging_obj.carry_failed_attempt_usage() + backup_cost: Final = await serve_first_call_then_fail_its_follow_up( + Usage(prompt_tokens=31, completion_tokens=5, total_tokens=36) ) + lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": logging_obj}) + usage: Final = lifted["combined_usage_object"] - assert (usage.prompt_tokens, usage.completion_tokens) == (559, 113) - assert first_call_cost > 0 - assert lifted["response_cost"] == pytest.approx(first_call_cost) + assert (usage.prompt_tokens, usage.completion_tokens) == (559 + 31, 113 + 5) + assert primary_cost > 0 and backup_cost > 0 + assert lifted["response_cost"] == pytest.approx(primary_cost + backup_cost) @pytest.mark.parametrize( diff --git a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py index af9a0b50805..07efd201523 100644 --- a/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py +++ b/tests/unit/responses/mcp/test_aresponses_api_with_mcp.py @@ -26,7 +26,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.xai.common_utils import xai_reported_cost_in_usd from litellm.proxy.mcp_server import mcp_server_manager from litellm.proxy.mcp_server.faults.list_outcomes import ServerListOk -from litellm.proxy.utils import _failure_fields_to_lift +from litellm.proxy.utils import _estimate_dispatched_failure_usage, _failure_fields_to_lift from litellm.responses.mcp.litellm_proxy_mcp_handler import LiteLLM_Proxy_MCP_Handler from litellm.responses.mcp.mcp_streaming_iterator import MCPEnhancedStreamingIterator from litellm.types.llms.openai import ( @@ -540,13 +540,9 @@ async def test_mcp_allowed_tools_filtering(): ] # First filter by allowed tools - filtered_tools_with_duplicates = ( - LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools( - mcp_tools=mock_mcp_tools_with_duplicates, - mcp_tools_with_litellm_proxy=cast( - List[ToolParam], mcp_tool_config_with_duplicates - ), - ) + filtered_tools_with_duplicates = LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools( + mcp_tools=mock_mcp_tools_with_duplicates, + mcp_tools_with_litellm_proxy=cast(List[ToolParam], mcp_tool_config_with_duplicates), ) # Then deduplicate the filtered tools @@ -605,11 +601,9 @@ async def test_mcp_allowed_tools_filtering(): }, ] - filtered_tools_multiple = ( - LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools( - mcp_tools=mock_mcp_tools_from_server, - mcp_tools_with_litellm_proxy=cast(List[ToolParam], multiple_mcp_configs), - ) + filtered_tools_multiple = LiteLLM_Proxy_MCP_Handler._filter_mcp_tools_by_allowed_tools( + mcp_tools=mock_mcp_tools_from_server, + mcp_tools_with_litellm_proxy=cast(List[ToolParam], multiple_mcp_configs), ) # Should return union of all allowed tools (3 unique tools) @@ -1918,7 +1912,7 @@ def _router_retrying_once() -> litellm.Router: @pytest.mark.asyncio -async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_each_attempt_its_own_rounds( +async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_the_failed_attempts_first_round_on_success( respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch ): _serve_responses( @@ -1945,9 +1939,11 @@ async def test_non_stream_auto_execute_retried_after_a_failed_follow_up_bills_ea assert len(recorder.logged) == 1 row: Final = recorder.logged[0] assert row["id"] == response.id - assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149, 40 + 5) - assert (row["prompt_tokens"], row["completion_tokens"]) == (response.usage.input_tokens, response.usage.output_tokens) - assert row["response_cost"] == pytest.approx(_round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY)) + assert (response.usage.input_tokens, response.usage.output_tokens) == (500 + 149, 40 + 5) + assert (row["prompt_tokens"], row["completion_tokens"]) == (500 + 149 + 71, 40 + 5 + 19) + assert row["response_cost"] == pytest.approx( + _round_cost(_RETRIED_ROUND_ONE) + _round_cost(_ROUND_TWO_BODY) + _round_cost(_ROUND_ONE_BODY) + ) @pytest.mark.asyncio @@ -1982,7 +1978,22 @@ async def test_non_stream_auto_execute_failing_every_retry_bills_each_paid_first failure_row_fields: Final = _failure_fields_to_lift(MappingProxyType({"litellm_logging_obj": logging_obj})) billed_usage: Final = failure_row_fields["combined_usage_object"] assert isinstance(billed_usage, Usage) - assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == billed_tokens + estimated_failure_usage: Final = ( + _estimate_dispatched_failure_usage( + model="gpt-5.6", + request_input=logging_obj.model_call_details.get("messages"), + system_input=None, + ) + if not retried_first_rounds + else None + ) + expected_prompt_tokens: Final = billed_tokens[0] + ( + estimated_failure_usage.prompt_tokens if estimated_failure_usage is not None else 0 + ) + assert (billed_usage.prompt_tokens, billed_usage.completion_tokens) == ( + expected_prompt_tokens, + billed_tokens[1], + ) assert failure_row_fields["response_cost"] == pytest.approx( _round_cost(_ROUND_ONE_BODY) + sum(_round_cost(body) for body in retried_first_rounds) ) diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index dc7e8d64889..94708f5841a 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -3652,14 +3652,28 @@ class _CutSSE(httpx.AsyncByteStream): raise httpx.RemoteProtocolError("peer closed connection without sending complete message body") -_BACKUP_SSE: Final = "".join( - f"data: {json.dumps(chunk)}\n\n" - for chunk in ( - {"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest", - "choices": [{"index": 0, "delta": {"role": "assistant", "content": " self-attention."}, "finish_reason": None}]}, - {"id": "b", "object": "chat.completion.chunk", "created": 1700000000, "model": "mistral-large-latest", - "choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}], - "usage": {"prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96}}, +_BACKUP_SSE: Final = ( + "".join( + f"data: {json.dumps(chunk)}\n\n" + for chunk in ( + { + "id": "b", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "mistral-large-latest", + "choices": [ + {"index": 0, "delta": {"role": "assistant", "content": " self-attention."}, "finish_reason": None} + ], + }, + { + "id": "b", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "mistral-large-latest", + "choices": [{"index": 0, "delta": {"content": ""}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 91, "completion_tokens": 5, "total_tokens": 96}, + }, + ) ) ) + "data: [DONE]\n\n" @@ -3907,88 +3921,165 @@ class _ScriptedAgenticFollowUps(CustomLogger): self.shared_call_succeeded.set() -async def _route_agentic_request_with_retries( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...] +_AGENTIC_BACKUP_USAGE: Final = {"prompt_tokens": 20, "completion_tokens": 3, "total_tokens": 23} +_AGENTIC_BACKUP_BODY: Final = { + "id": "chatcmpl-backup", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "42"}}], + "usage": _AGENTIC_BACKUP_USAGE, +} +_AGENTIC_RATE_LIMITED: Final = { + "error": {"message": "Rate limit reached", "type": "requests", "code": "rate_limit_exceeded"} +} + + +async def _route_agentic_request( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + follow_ups: tuple[str, ...], + model: str = "m", + first_call_replies: tuple[httpx.Response, ...] = (), + fallbacks: tuple[str, ...] = (), ) -> tuple[object, LiteLLMLogging, respx.Route, _ScriptedAgenticFollowUps]: monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) hook: Final = _ScriptedAgenticFollowUps(follow_ups) monkeypatch.setattr(litellm, "callbacks", [hook]) - first_calls: Final = respx_mock.post("https://api.openai.com/v1/chat/completions").mock( - return_value=httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY) + first_calls: Final = ( + respx_mock.post("https://api.openai.com/v1/chat/completions").mock(side_effect=list(first_call_replies)) + if first_call_replies + else respx_mock.post("https://api.openai.com/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY) + ) + ) + respx_mock.post("https://backup.test/v1/chat/completions").mock( + return_value=httpx.Response(200, json=_AGENTIC_BACKUP_BODY) ) router: Final = Router( - model_list=[{"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}], + model_list=[ + {"model_name": "m", "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "sk-test"}}, + { + "model_name": "backup", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-test", + "api_base": "https://backup.test/v1", + }, + }, + ], num_retries=2, retry_after=0, + fallbacks=[{"m": list(fallbacks)}] if fallbacks else [], ) messages: Final = [{"role": "user", "content": "what is 6*7?"}] shared_logging, _ = litellm.utils.function_setup( original_function="acompletion", rules_obj=litellm.utils.Rules(), start_time=datetime.now(), - model="m", + model=model, messages=messages, litellm_call_id="shared-agentic-call", ) try: outcome: object = await router.acompletion( - model="m", messages=messages, litellm_call_id="shared-agentic-call", litellm_logging_obj=shared_logging + model=model, messages=messages, litellm_call_id="shared-agentic-call", litellm_logging_obj=shared_logging ) except litellm.RateLimitError as error: outcome = error return outcome, shared_logging, first_calls, hook +def _gpt_4o_mini_cost(usage: Mapping[str, int]) -> float: + return sum( + litellm.cost_per_token( + model="gpt-4o-mini", prompt_tokens=usage["prompt_tokens"], completion_tokens=usage["completion_tokens"] + ) + ) + + @pytest.mark.asyncio -async def test_retried_agentic_request_whose_follow_up_always_fails_bills_every_first_call( +async def test_agentic_request_whose_follow_up_fails_is_not_retried_and_bills_its_one_first_call( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter ) -> None: from litellm.proxy.utils import _failure_fields_to_lift - outcome, shared_logging, first_calls, _ = await _route_agentic_request_with_retries( + outcome, shared_logging, first_calls, _ = await _route_agentic_request( monkeypatch, respx_mock, ("litellm.RateLimitError",) ) lifted: Final = _failure_fields_to_lift({"litellm_logging_obj": shared_logging}) - first_call_cost: Final = sum( - litellm.cost_per_token(model="gpt-4o-mini", prompt_tokens=50, completion_tokens=7) - ) assert isinstance(outcome, litellm.RateLimitError) - assert first_calls.call_count == 3 - assert lifted["combined_usage_object"].prompt_tokens == 3 * _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"] - assert lifted["combined_usage_object"].completion_tokens == 3 * _AGENTIC_FIRST_CALL_USAGE["completion_tokens"] - assert lifted["response_cost"] == pytest.approx(3 * first_call_cost) - - -async def _successful_agentic_request_cost_report( - monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, follow_ups: tuple[str, ...] -) -> tuple[int, Mapping[str, object], Mapping[str, str | None]]: - from litellm.proxy.common_request_processing import cost_headers - - outcome, shared_logging, _, hook = await _route_agentic_request_with_retries(monkeypatch, respx_mock, follow_ups) - await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10) - assert isinstance(outcome, litellm.ModelResponse) - success: Final = shared_logging.model_call_details["standard_logging_object"] - billed: Final = { - field: success[field] for field in ("response_cost", "prompt_tokens", "completion_tokens", "cost_breakdown") - } - return hook.plans_built, billed, cost_headers(success["response_cost"], shared_logging) + assert outcome.status_code == 429 + assert first_calls.call_count == 1 + assert lifted["combined_usage_object"].prompt_tokens == _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"] + assert lifted["combined_usage_object"].completion_tokens == _AGENTIC_FIRST_CALL_USAGE["completion_tokens"] + assert lifted["response_cost"] == pytest.approx(_gpt_4o_mini_cost(_AGENTIC_FIRST_CALL_USAGE)) @pytest.mark.asyncio -async def test_retried_agentic_request_that_recovers_reports_the_cost_of_a_first_try_success( +async def test_agentic_request_whose_first_call_fails_is_still_retried( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter ) -> None: - first_try_attempts, first_try_billed, first_try_headers = await _successful_agentic_request_cost_report( - monkeypatch, respx_mock, ("42",) - ) - recovered_attempts, recovered_billed, recovered_headers = await _successful_agentic_request_cost_report( - monkeypatch, respx_mock, ("litellm.RateLimitError", "42") + outcome, _, first_calls, hook = await _route_agentic_request( + monkeypatch, + respx_mock, + ("42",), + first_call_replies=( + httpx.Response(429, json=_AGENTIC_RATE_LIMITED), + httpx.Response(200, json=_AGENTIC_FIRST_CALL_BODY), + ), ) - assert (first_try_attempts, recovered_attempts) == (1, 2) - assert recovered_billed == first_try_billed - assert recovered_headers == first_try_headers + assert isinstance(outcome, litellm.ModelResponse) + assert first_calls.call_count == 2 + assert hook.plans_built == 1 + + +async def _successful_agentic_request_billing( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + follow_ups: tuple[str, ...], + model: str, + fallbacks: tuple[str, ...] = (), +) -> tuple[Mapping[str, object], Mapping[str, str | None]]: + from litellm.proxy.common_request_processing import cost_headers + + outcome, shared_logging, _, hook = await _route_agentic_request( + monkeypatch, respx_mock, follow_ups, model=model, fallbacks=fallbacks + ) + await asyncio.wait_for(hook.shared_call_succeeded.wait(), timeout=10) + assert isinstance(outcome, litellm.ModelResponse) + success: Final = shared_logging.model_call_details["standard_logging_object"] + return success, cost_headers(success["response_cost"], shared_logging) + + +@pytest.mark.asyncio +async def test_fallback_after_a_failed_agentic_follow_up_bills_the_primarys_first_call_on_the_success_row( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + first_try, first_try_headers = await _successful_agentic_request_billing( + monkeypatch, respx_mock, ("42",), model="backup" + ) + recovered, recovered_headers = await _successful_agentic_request_billing( + monkeypatch, respx_mock, ("litellm.RateLimitError",), model="m", fallbacks=("backup",) + ) + + first_call_cost: Final = _gpt_4o_mini_cost(_AGENTIC_FIRST_CALL_USAGE) + assert first_try["response_cost"] == pytest.approx(_gpt_4o_mini_cost(_AGENTIC_BACKUP_USAGE)) + assert "failed_attempts_cost" not in first_try["cost_breakdown"].get("additional_costs", {}) + assert recovered["response_cost"] == pytest.approx(first_try["response_cost"] + first_call_cost) + assert recovered["prompt_tokens"] == first_try["prompt_tokens"] + _AGENTIC_FIRST_CALL_USAGE["prompt_tokens"] + assert ( + recovered["completion_tokens"] + == first_try["completion_tokens"] + _AGENTIC_FIRST_CALL_USAGE["completion_tokens"] + ) + assert recovered["cost_breakdown"]["total_cost"] == pytest.approx(recovered["response_cost"]) + assert recovered["cost_breakdown"]["additional_costs"]["failed_attempts_cost"] == pytest.approx(first_call_cost) + assert float(recovered_headers["x-litellm-response-cost"]) == pytest.approx(recovered["response_cost"]) + assert {key: value for key, value in recovered_headers.items() if key != "x-litellm-response-cost"} == { + key: value for key, value in first_try_headers.items() if key != "x-litellm-response-cost" + } def _responses_sse(*events: Mapping[str, object]) -> str: @@ -13817,7 +13908,11 @@ def test_get_search_tool_support_reports_each_routable_deployment_of_the_group() @pytest.mark.parametrize("provider", ["chatgpt", "github_copilot"]) @pytest.mark.parametrize("explicit_provider", [False, True]) async def test_search_tool_catalog_lookup_needs_no_credentials( - provider: str, explicit_provider: bool, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter + provider: str, + explicit_provider: bool, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, ) -> None: monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "chatgpt")) monkeypatch.setenv("GITHUB_COPILOT_TOKEN_DIR", str(tmp_path / "copilot")) @@ -17485,13 +17580,16 @@ def test_model_group_info_fast_mode_fails_closed_without_explicit_boolean( def test_model_group_info_fast_mode_requires_every_deployment( local_model_cost_map: None, other_model: str, expected: bool, reverse: bool ) -> None: - models: Final = (other_model, "anthropic/claude-opus-5") if reverse else ( - "anthropic/claude-opus-5", other_model + models: Final = (other_model, "anthropic/claude-opus-5") if reverse else ("anthropic/claude-opus-5", other_model) + router: Final = Router( + model_list=[ + { + "model_name": "fast-group", + "litellm_params": {"model": model, "api_key": "fake-key"}, + } + for model in models + ] ) - router: Final = Router(model_list=[{ - "model_name": "fast-group", - "litellm_params": {"model": model, "api_key": "fake-key"}, - } for model in models]) result: Final = router.get_model_group_info("fast-group") @@ -17849,10 +17947,7 @@ class _AnthropicMessagesFallbackByteStream: def _anthropic_messages_overloaded_error_chunk() -> bytes: - return ( - b"event: error\n" - b'data: {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}\n\n' - ) + return b'event: error\ndata: {"type": "error", "error": {"type": "overloaded_error", "message": "Overloaded"}}\n\n' def _anthropic_messages_invalid_request_error_chunk() -> bytes: @@ -24600,16 +24695,25 @@ class TestAutoRouterTraceProvenance: monkeypatch.setattr(litellm, "_async_success_callback", [logger, CallbackComplete()]) classifier: Final = ObservedClassifier() - router: Final = Router(model_list=[ - {**self._marker("blue"), "litellm_params": { - "model": "auto_router/complexity_router", - "complexity_router_config": { - "classifier_type": "custom", "classifier_plugin": classifier, - "tiers": {"SIMPLE": "answer"}, "session_affinity": False, "deployment_affinity": False, + router: Final = Router( + model_list=[ + { + **self._marker("blue"), + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": { + "classifier_type": "custom", + "classifier_plugin": classifier, + "tiers": {"SIMPLE": "answer"}, + "session_affinity": False, + "deployment_affinity": False, + }, + }, }, - }}, - {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, - ], plugins=[classifier]) + {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, + ], + plugins=[classifier], + ) stale: Final = {"routing_decision": {"router_model_name": "previous"}, "_autorouter_baseline_route": "old"} request: Final = {"metadata": {**stale}, "litellm_metadata": {**stale, "user_api_key_team_id": "team"}} with logger.tracer.start_as_current_span("request") as root: @@ -24694,20 +24798,22 @@ class TestAutoRouterTraceProvenance: self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]" ) -> None: logger, exporter = _tracing - router: Final = Router(model_list=[ - self._marker("blue"), - { - "model_name": "traced-router", - "model_info": {"id": "definition-semantic"}, - "litellm_params": { - "model": "auto_router/semantic-router", - "tags": ["blue"], - "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), - "auto_router_default_model": "answer", - "auto_router_embedding_model": "embed", + router: Final = Router( + model_list=[ + self._marker("blue"), + { + "model_name": "traced-router", + "model_info": {"id": "definition-semantic"}, + "litellm_params": { + "model": "auto_router/semantic-router", + "tags": ["blue"], + "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), + "auto_router_default_model": "answer", + "auto_router_embedding_model": "embed", + }, }, - }, - ]) + ] + ) with logger.start_phase_span("route traced-router"): response: Final = await router.async_pre_routing_hook( model="traced-router", request_kwargs={"metadata": {"tags": ["blue"]}} @@ -24727,26 +24833,42 @@ class TestAutoRouterTraceProvenance: ) -> None: logger, exporter = _tracing monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) - respx_mock.post("https://embedding.example/v1/embeddings").respond(200, json={ - "object": "list", - "model": "text-embedding-3-small", - "data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}], - "usage": {"prompt_tokens": 1, "total_tokens": 1}, - }) - router: Final = Router(model_list=[ - { - "model_name": "internal-semantic-router", - "model_info": {"id": "semantic-definition", "team_id": "trace-team", "team_public_model_name": "public-router"}, - "litellm_params": { - "model": "auto_router/semantic-router", - "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), - "auto_router_default_model": "answer", - "auto_router_embedding_model": "embed", - }, + respx_mock.post("https://embedding.example/v1/embeddings").respond( + 200, + json={ + "object": "list", + "model": "text-embedding-3-small", + "data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}], + "usage": {"prompt_tokens": 1, "total_tokens": 1}, }, - {"model_name": "embed", "litellm_params": {"model": "openai/text-embedding-3-small", "api_base": "https://embedding.example/v1", "api_key": "test"}}, - {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, - ]) + ) + router: Final = Router( + model_list=[ + { + "model_name": "internal-semantic-router", + "model_info": { + "id": "semantic-definition", + "team_id": "trace-team", + "team_public_model_name": "public-router", + }, + "litellm_params": { + "model": "auto_router/semantic-router", + "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}), + "auto_router_default_model": "answer", + "auto_router_embedding_model": "embed", + }, + }, + { + "model_name": "embed", + "litellm_params": { + "model": "openai/text-embedding-3-small", + "api_base": "https://embedding.example/v1", + "api_key": "test", + }, + }, + {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}}, + ] + ) request: Final = {"metadata": {"user_api_key_team_id": "trace-team"}} with logger.tracer.start_as_current_span("request"): await router.async_get_available_deployment( From ef28e097bbee933fe46bc3776c291245afd8b472 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 01:58:06 -0700 Subject: [PATCH 09/12] fix(code_interpreter): delete a session sandbox whose code run failed so the next call starts a fresh one --- .../code_interpreter_interception/handler.py | 8 ++++ .../test_handler.py | 47 +++++++++++++++++++ 2 files changed, 55 insertions(+) diff --git a/litellm/integrations/code_interpreter_interception/handler.py b/litellm/integrations/code_interpreter_interception/handler.py index 950088ff9e0..c83262dbb78 100644 --- a/litellm/integrations/code_interpreter_interception/handler.py +++ b/litellm/integrations/code_interpreter_interception/handler.py @@ -529,6 +529,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): except Exception: await self._delete_container_for_cache_key(sandbox_key) raise + await self._evict_sandbox_if_a_run_failed(sandbox_key, code_interpreter_calls) optional_params: Final = anthropic_messages_optional_request_params request_patch: Final = AgenticLoopRequestPatch( @@ -583,6 +584,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): raise tool_messages: Final = [result[0] for result in tool_results] code_interpreter_calls: Final = [result[1] for result in tool_results] + await self._evict_sandbox_if_a_run_failed(sandbox_key, code_interpreter_calls) request_patch: Final = AgenticLoopRequestPatch( model=model, @@ -736,6 +738,12 @@ class CodeInterpreterInterceptionLogger(CustomLogger): except (json.JSONDecodeError, TypeError, AttributeError): return "" + async def _evict_sandbox_if_a_run_failed( + self, sandbox_key: str | None, calls: Sequence[CodeInterpreterCall] + ) -> None: + if any(call["status"] == "failed" for call in calls): + await self._delete_container_for_cache_key(sandbox_key) + async def _sandbox_tool_output(self, sandbox: _Sandbox, arguments: str) -> _ToolRun: match sandbox: case _SandboxUnavailable(): diff --git a/tests/unit/integrations/code_interpreter_interception/test_handler.py b/tests/unit/integrations/code_interpreter_interception/test_handler.py index 1c988a35fdc..59b2fb2f676 100644 --- a/tests/unit/integrations/code_interpreter_interception/test_handler.py +++ b/tests/unit/integrations/code_interpreter_interception/test_handler.py @@ -1511,6 +1511,53 @@ async def test_a_sandbox_failure_answers_the_chat_tool_call_with_a_fixed_failure assert [call["status"] for call in plan.metadata["code_interpreter_calls"]] == ["failed"] +_SESSION_PLAN_SURFACES: Final = ( + pytest.param({}, FakeResponse(output=[_function_call_item()]), id="responses"), + pytest.param( + {_ACTIVE_KEY: True, "_agentic_loop_api_surface": CHAT_COMPLETION_AGENTIC_SURFACE}, + {"choices": [{"message": {"tool_calls": [_chat_function_call_item()]}}]}, + id="chat", + ), +) + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("surface_kwargs", "response"), _SESSION_PLAN_SURFACES) +async def test_a_session_sandbox_whose_code_run_fails_is_not_reused_by_the_next_request(surface_kwargs, response): + sandbox: Final = _SandboxThatCannotRunCode() + logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) + + for request_id in ("req1", "req2"): + await logger.async_build_agentic_loop_plan( + tools={ + "tool_calls": [ + { + "id": "call_1", + "call_id": "c1", + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "arguments": '{"code":"print(1)"}', + } + ] + }, + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + response=response, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={"tools": []}, + logging_obj=FakeLogging(litellm_call_id=request_id), + stream=False, + kwargs={ + "litellm_call_id": request_id, + _SANDBOX_KEY: "conv-1", + _SESSION_SCOPED_KEY: True, + **surface_kwargs, + }, + ) + + assert len(sandbox.create_calls) == 2 + assert len(sandbox.delete_calls) == 2 + + def _anthropic_message(content: list[dict[str, object]], stop_reason: str) -> dict[str, object]: return { "id": "msg_1", From c056772ea36a868deea54292b479789710e481cf Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:18:06 -0700 Subject: [PATCH 10/12] fix(code_interpreter): delete only the session sandbox that failed, never its replacement --- .../code_interpreter_interception/handler.py | 17 +++- .../test_handler.py | 97 ++++++++++++++----- 2 files changed, 83 insertions(+), 31 deletions(-) diff --git a/litellm/integrations/code_interpreter_interception/handler.py b/litellm/integrations/code_interpreter_interception/handler.py index c83262dbb78..e204fbd7f62 100644 --- a/litellm/integrations/code_interpreter_interception/handler.py +++ b/litellm/integrations/code_interpreter_interception/handler.py @@ -529,7 +529,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): except Exception: await self._delete_container_for_cache_key(sandbox_key) raise - await self._evict_sandbox_if_a_run_failed(sandbox_key, code_interpreter_calls) + await self._evict_sandbox_if_a_run_failed(sandbox_key, sandbox, code_interpreter_calls) optional_params: Final = anthropic_messages_optional_request_params request_patch: Final = AgenticLoopRequestPatch( @@ -584,7 +584,7 @@ class CodeInterpreterInterceptionLogger(CustomLogger): raise tool_messages: Final = [result[0] for result in tool_results] code_interpreter_calls: Final = [result[1] for result in tool_results] - await self._evict_sandbox_if_a_run_failed(sandbox_key, code_interpreter_calls) + await self._evict_sandbox_if_a_run_failed(sandbox_key, sandbox, code_interpreter_calls) request_patch: Final = AgenticLoopRequestPatch( model=model, @@ -739,10 +739,17 @@ class CodeInterpreterInterceptionLogger(CustomLogger): return "" async def _evict_sandbox_if_a_run_failed( - self, sandbox_key: str | None, calls: Sequence[CodeInterpreterCall] + self, sandbox_key: str | None, sandbox: _Sandbox, calls: Sequence[CodeInterpreterCall] ) -> None: - if any(call["status"] == "failed" for call in calls): - await self._delete_container_for_cache_key(sandbox_key) + if not sandbox_key or not isinstance(sandbox, _ReadySandbox): + return + if not any(call["status"] == "failed" for call in calls): + return + cached: Final = self._container_cache.get(sandbox_key) + if cached is None or cached[0] is not sandbox.container: + return + self._container_cache.pop(sandbox_key) + await self._delete_container(container=cached[0], params=cached[1]) async def _sandbox_tool_output(self, sandbox: _Sandbox, arguments: str) -> _ToolRun: match sandbox: diff --git a/tests/unit/integrations/code_interpreter_interception/test_handler.py b/tests/unit/integrations/code_interpreter_interception/test_handler.py index 59b2fb2f676..47dbea8849d 100644 --- a/tests/unit/integrations/code_interpreter_interception/test_handler.py +++ b/tests/unit/integrations/code_interpreter_interception/test_handler.py @@ -5,6 +5,7 @@ All sandbox dependencies are injected: a FakeSandbox stands in for the real e2b config and records how it is called. """ +import asyncio import time from typing import Final @@ -1521,43 +1522,87 @@ _SESSION_PLAN_SURFACES: Final = ( ) +async def _build_session_plan(logger, request_id, surface_kwargs=None, response=None): + return await logger.async_build_agentic_loop_plan( + tools={ + "tool_calls": [ + { + "id": "call_1", + "call_id": "c1", + "name": LITELLM_CODE_EXECUTION_TOOL_NAME, + "arguments": '{"code":"print(1)"}', + } + ] + }, + model="gpt-5", + messages=[{"role": "user", "content": "x"}], + response=response or FakeResponse(output=[_function_call_item()]), + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={"tools": []}, + logging_obj=FakeLogging(litellm_call_id=request_id), + stream=False, + kwargs={ + "litellm_call_id": request_id, + _SANDBOX_KEY: "conv-1", + _SESSION_SCOPED_KEY: True, + **(surface_kwargs or {}), + }, + ) + + @pytest.mark.asyncio @pytest.mark.parametrize(("surface_kwargs", "response"), _SESSION_PLAN_SURFACES) async def test_a_session_sandbox_whose_code_run_fails_is_not_reused_by_the_next_request(surface_kwargs, response): sandbox: Final = _SandboxThatCannotRunCode() logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) - for request_id in ("req1", "req2"): - await logger.async_build_agentic_loop_plan( - tools={ - "tool_calls": [ - { - "id": "call_1", - "call_id": "c1", - "name": LITELLM_CODE_EXECUTION_TOOL_NAME, - "arguments": '{"code":"print(1)"}', - } - ] - }, - model="gpt-5", - messages=[{"role": "user", "content": "x"}], - response=response, - anthropic_messages_provider_config=None, - anthropic_messages_optional_request_params={"tools": []}, - logging_obj=FakeLogging(litellm_call_id=request_id), - stream=False, - kwargs={ - "litellm_call_id": request_id, - _SANDBOX_KEY: "conv-1", - _SESSION_SCOPED_KEY: True, - **surface_kwargs, - }, - ) + await _build_session_plan(logger, "req1", surface_kwargs, response) + await _build_session_plan(logger, "req2", surface_kwargs, response) assert len(sandbox.create_calls) == 2 assert len(sandbox.delete_calls) == 2 +class _SandboxWhoseFirstContainerDies(FakeSandbox): + def __init__(self): + super().__init__() + self.handles = [] + self.first_run_started = asyncio.Event() + self.first_run_may_finish = asyncio.Event() + + async def acreate_sandbox(self, **kwargs): + self.create_calls.append(kwargs) + handle = FakeHandle(sandbox_id=f"sbx_{len(self.handles)}") + self.handles.append(handle) + return handle + + async def arun_code(self, *, container, code, **kwargs): + self.run_calls.append({"container": container, "code": code}) + if len(self.run_calls) == 1: + self.first_run_started.set() + await self.first_run_may_finish.wait() + if container is self.handles[0]: + raise TimeoutError("sandbox expired") + return CodeExecutionResult(stdout=self.stdout) + + +@pytest.mark.asyncio +async def test_a_late_failure_on_a_dead_session_sandbox_keeps_its_healthy_replacement(): + sandbox: Final = _SandboxWhoseFirstContainerDies() + logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) + + slow_request: Final = asyncio.create_task(_build_session_plan(logger, "req-slow")) + await sandbox.first_run_started.wait() + await _build_session_plan(logger, "req-fails-fast") + await _build_session_plan(logger, "req-gets-replacement") + sandbox.first_run_may_finish.set() + await slow_request + await _build_session_plan(logger, "req-after") + + assert [call["container"] for call in sandbox.delete_calls] == [sandbox.handles[0]] + assert [call["container"] for call in sandbox.run_calls[-2:]] == [sandbox.handles[1], sandbox.handles[1]] + + def _anthropic_message(content: list[dict[str, object]], stop_reason: str) -> dict[str, object]: return { "id": "msg_1", From 1531d33a86ef02078a40e1fae6d1533e6f23f73d Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 02:32:56 -0700 Subject: [PATCH 11/12] test(code_interpreter): type the session sandbox test helpers --- .../test_handler.py | 25 +++++++++++++------ 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/tests/unit/integrations/code_interpreter_interception/test_handler.py b/tests/unit/integrations/code_interpreter_interception/test_handler.py index 47dbea8849d..c68bcf16154 100644 --- a/tests/unit/integrations/code_interpreter_interception/test_handler.py +++ b/tests/unit/integrations/code_interpreter_interception/test_handler.py @@ -7,6 +7,7 @@ config and records how it is called. import asyncio import time +from collections.abc import Mapping from typing import Final import httpx @@ -29,6 +30,7 @@ from litellm.llms.base_llm.sandbox.transformation import CodeExecutionResult from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, CODE_INTERPRETER_STREAM_OPTIONS_KEY, + AgenticLoopPlan, NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, is_interception_internal_key, ) @@ -1522,7 +1524,12 @@ _SESSION_PLAN_SURFACES: Final = ( ) -async def _build_session_plan(logger, request_id, surface_kwargs=None, response=None): +async def _build_session_plan( + logger: CodeInterpreterInterceptionLogger, + request_id: str, + surface_kwargs: Mapping[str, object] | None = None, + response: object | None = None, +) -> AgenticLoopPlan: return await logger.async_build_agentic_loop_plan( tools={ "tool_calls": [ @@ -1552,7 +1559,9 @@ async def _build_session_plan(logger, request_id, surface_kwargs=None, response= @pytest.mark.asyncio @pytest.mark.parametrize(("surface_kwargs", "response"), _SESSION_PLAN_SURFACES) -async def test_a_session_sandbox_whose_code_run_fails_is_not_reused_by_the_next_request(surface_kwargs, response): +async def test_a_session_sandbox_whose_code_run_fails_is_not_reused_by_the_next_request( + surface_kwargs: Mapping[str, object], response: object +) -> None: sandbox: Final = _SandboxThatCannotRunCode() logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) @@ -1564,19 +1573,19 @@ async def test_a_session_sandbox_whose_code_run_fails_is_not_reused_by_the_next_ class _SandboxWhoseFirstContainerDies(FakeSandbox): - def __init__(self): + def __init__(self) -> None: super().__init__() - self.handles = [] + self.handles: list[FakeHandle] = [] self.first_run_started = asyncio.Event() self.first_run_may_finish = asyncio.Event() - async def acreate_sandbox(self, **kwargs): + async def acreate_sandbox(self, **kwargs: object) -> FakeHandle: self.create_calls.append(kwargs) - handle = FakeHandle(sandbox_id=f"sbx_{len(self.handles)}") + handle: Final = FakeHandle(sandbox_id=f"sbx_{len(self.handles)}") self.handles.append(handle) return handle - async def arun_code(self, *, container, code, **kwargs): + async def arun_code(self, *, container: FakeHandle, code: str, **kwargs: object) -> CodeExecutionResult: self.run_calls.append({"container": container, "code": code}) if len(self.run_calls) == 1: self.first_run_started.set() @@ -1587,7 +1596,7 @@ class _SandboxWhoseFirstContainerDies(FakeSandbox): @pytest.mark.asyncio -async def test_a_late_failure_on_a_dead_session_sandbox_keeps_its_healthy_replacement(): +async def test_a_late_failure_on_a_dead_session_sandbox_keeps_its_healthy_replacement() -> None: sandbox: Final = _SandboxWhoseFirstContainerDies() logger: Final = CodeInterpreterInterceptionLogger(sandbox_config=sandbox) From bc95cab434741cb750eef43429c42fb32ede94cb Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 31 Oct 2026 03:48:37 -0700 Subject: [PATCH 12/12] fix(proxy): report what a failed agentic follow-up's failure row bills on /v1/messages errors --- .../proxy/anthropic_endpoints/endpoints.py | 6 +- .../anthropic_endpoints/test_endpoints.py | 81 +++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 0e204cf9211..0ec11c66fe2 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -18,6 +18,7 @@ from litellm.anthropic_interface.exceptions import ( AnthropicExceptionMapping, ) from litellm.integrations.custom_guardrail import ModifyResponseException +from litellm.litellm_core_utils.agentic_followup_failure import is_agentic_followup_failure from litellm.litellm_core_utils.tool_fields import first_validation_error from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler from litellm.llms.anthropic.messages.context_management import ( @@ -49,6 +50,7 @@ from litellm.proxy.common_utils.openai_error_payload import ( ) from litellm.proxy.common_utils.token_count_log import rejected_token_count from litellm.proxy.common_utils.token_count_settings import count_source_fields +from litellm.proxy.hooks.proxy_track_cost_callback import failure_row_response_cost from litellm.proxy.litellm_pre_call_utils import ( forwarded_anthropic_oauth_token, should_auto_drop_params_for_agentic_cli, @@ -284,7 +286,9 @@ async def anthropic_response( call_id=base_llm_response_processor.litellm_call_id, model_id=model_id, version=version, - response_cost=0, + response_cost=( + failure_row_response_cost(base_llm_response_processor.data) if is_agentic_followup_failure(e) else 0 + ), model_region=getattr(user_api_key_dict, "allowed_model_region", ""), request_data=data, timeout=getattr(e, "timeout", None), diff --git a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py index 604fc84a096..79e41312905 100644 --- a/tests/unit/proxy/anthropic_endpoints/test_endpoints.py +++ b/tests/unit/proxy/anthropic_endpoints/test_endpoints.py @@ -2,6 +2,7 @@ Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objects in streaming responses """ +import datetime import json import logging import unittest @@ -248,6 +249,86 @@ class TestBlockedResponseUsage: assert response["content"] == [{"type": "text", "text": _BLOCK_TEXT}] +class TestAnthropicResponseReportsWhatAFailedAgenticFollowUpBills: + async def _error_response_cost(self, raise_the_follow_up_error: bool) -> float: + import litellm + import litellm.proxy.anthropic_endpoints.endpoints as ep + import litellm.proxy.proxy_server as proxy_server + from litellm.litellm_core_utils.chat_completion_agentic_loop import bill_first_round_on_failure + from litellm.proxy.proxy_server import ProxyConfig + from litellm.proxy.utils import ProxyLogging + from litellm.types.utils import ModelResponse, Usage + + proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging.alert_types = [] + follow_up_error: Final = litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o") + + async def failing_follow_up() -> ModelResponse: + raise follow_up_error + + async def fail_after_the_first_call(self: ProxyBaseLLMRequestProcessing, **kwargs: object) -> None: + _, logging_obj = await self.common_processing_pre_call_logic( + request=kwargs["request"], + general_settings={}, + user_api_key_dict=kwargs["user_api_key_dict"], + proxy_logging_obj=proxy_logging, + proxy_config=MagicMock(spec=ProxyConfig), + route_type="anthropic_messages", + ) + logging_obj.model_call_details.update(first_api_call_start_time=datetime.datetime.now()) + with pytest.raises(litellm.RateLimitError): + await bill_first_round_on_failure( + failing_follow_up(), + logging_obj=logging_obj, + response=ModelResponse( + model="gpt-4o", usage=Usage(prompt_tokens=559, completion_tokens=113, total_tokens=672) + ), + stream=False, + ) + raise ( + follow_up_error + if raise_the_follow_up_error + else litellm.RateLimitError(message="rate limited", llm_provider="openai", model="gpt-4o") + ) + + request: Final = MagicMock(spec=Request) + request.headers = {} + with ( + patch.object( + ep, + "read_request_body", + new=AsyncMock( + return_value={ + "model": "gpt-4o", + "max_tokens": 64, + "messages": [{"role": "user", "content": "what is 6*7?"}], + } + ), + ), + patch.object(ep.ProxyBaseLLMRequestProcessing, "base_process_llm_request", new=fail_after_the_first_call), + patch.object(proxy_server, "proxy_logging_obj", proxy_logging), + ): + response: Final = await ep.anthropic_response( + fastapi_response=Response(), + request=request, + user_api_key_dict=UserAPIKeyAuth(request_route="/v1/messages"), + ) + + assert response.status_code == 429 + return float(response.headers["x-litellm-response-cost"]) + + @pytest.mark.asyncio + async def test_the_error_response_reports_the_served_first_call_its_failure_row_bills(self) -> None: + import litellm + + price: Final = litellm.model_cost["gpt-4o"] + + assert await self._error_response_cost(raise_the_follow_up_error=True) == pytest.approx( + 559 * price["input_cost_per_token"] + 113 * price["output_cost_per_token"] + ) + assert await self._error_response_cost(raise_the_follow_up_error=False) == 0 + + class TestProxyExceptionAnthropicEnvelope: @pytest.mark.asyncio async def test_anthropic_response_maps_proxy_exception_to_anthropic_envelope(self):