fix(agentic): end a capped agentic turn on chat and responses instead of a retried 500

This commit is contained in:
mateo-berri 2026-10-29 11:43:00 -07:00
parent b3849b29de
commit e9ce495628
7 changed files with 390 additions and 154 deletions

View file

@ -0,0 +1,72 @@
from collections.abc import Iterable
from itertools import chain
from typing import Final
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.utils import Choices, ModelResponse
def _strings(values: Iterable[object]) -> frozenset[str]:
return frozenset(value for value in values if isinstance(value, str))
def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
wrapped: Final = message_field(tool_calls, "tool_calls")
calls: Final = parts_of(tool_calls if wrapped is None else wrapped)
ids: Final = _strings(
chain.from_iterable((message_field(call, "id"), message_field(call, "call_id")) for call in calls)
)
names: Final = _strings(message_field(call, "name") for call in calls)
return ids, names
def is_refused_call(call_id: object, name: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
if isinstance(call_id, str) and refused_ids:
return call_id in refused_ids
return name in refused_names
def is_refused_item(
item: object, item_type: str, id_field: str, refused_ids: frozenset[str], refused_names: frozenset[str]
) -> bool:
if message_field(item, "type") != item_type:
return False
return is_refused_call(message_field(item, id_field), message_field(item, "name"), refused_ids, refused_names)
def finalize_refused_responses_turn(response: ResponsesAPIResponse, tool_calls: object) -> ResponsesAPIResponse:
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
kept: Final = [
item
for item in parts_of(message_field(response, "output"))
if not is_refused_item(item, "function_call", "call_id", refused_ids, refused_names)
]
return response.model_copy(update={"output": kept})
def _is_refused_chat_tool_call(call: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
name: Final = message_field(message_field(call, "function"), "name")
return is_refused_call(message_field(call, "id"), name, refused_ids, refused_names)
def _finalized_choice(choice: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> object:
if not isinstance(choice, Choices) or not choice.message.tool_calls:
return choice
kept: Final = [
call for call in choice.message.tool_calls if not _is_refused_chat_tool_call(call, refused_ids, refused_names)
]
finish_reason: Final = "stop" if not kept and choice.finish_reason == "tool_calls" else choice.finish_reason
return choice.model_copy(
update={
"message": choice.message.model_copy(update={"tool_calls": kept or None}),
"finish_reason": finish_reason,
}
)
def finalize_refused_chat_turn(response: ModelResponse, tool_calls: object) -> ModelResponse:
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
return response.model_copy(
update={"choices": [_finalized_choice(choice, refused_ids, refused_names) for choice in response.choices]}
)

View file

@ -11,6 +11,7 @@ from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
from litellm.litellm_core_utils.agentic_loop_refusal import finalize_refused_chat_turn
from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
@ -22,6 +23,7 @@ from litellm.types.integrations.custom_logger import (
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
AgenticLoopPlan,
AgenticLoopRequestPatch,
AgenticLoopSafetyError,
converted_stream_requested,
is_interception_internal_key,
stashed_stream_options,
@ -101,9 +103,9 @@ def _check_agentic_loop_safety(
) -> str:
fingerprint: Final = _fingerprint_tools(tool_calls)
if fingerprint in fingerprints:
raise ValueError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
if depth >= max_loops:
raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
return fingerprint
@ -132,6 +134,26 @@ def _converted_stream_replay(
)
def _end_refused_turn(
finalized: ModelResponse,
*,
kwargs: Mapping[str, object],
depth: int,
model: str,
custom_llm_provider: str,
logging_obj: object,
) -> ModelResponse | CustomStreamWrapper:
replay: Final = _converted_stream_replay(
finalized,
kwargs=kwargs,
depth=depth,
model=model,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
)
return finalized if replay is None else replay
def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]:
metadata: Final = kwargs_for_followup.get("litellm_metadata")
return MappingProxyType(
@ -296,13 +318,26 @@ async def maybe_run_chat_completion_agentic_loop(
if not should_run:
continue
fingerprint = _check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
try:
fingerprint = _check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
except AgenticLoopSafetyError as e:
if stream:
raise
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [model=%s]: %s", model, str(e))
return _end_refused_turn(
finalize_refused_chat_turn(response, tool_calls),
kwargs=kwargs,
depth=depth,
model=model,
custom_llm_provider=custom_llm_provider,
logging_obj=logging_obj,
)
if not _build_plan_overridden(callback):
return await callback.async_run_agentic_loop(

View file

@ -41,6 +41,11 @@ from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, up
from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.files.types import FileContentStreamingResult
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
from litellm.litellm_core_utils.agentic_loop_refusal import (
finalize_refused_responses_turn,
is_refused_item,
refused_tool_call_identifiers,
)
from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
@ -5864,26 +5869,6 @@ class BaseLLMHTTPHandler:
except Exception:
return str(tools)
@staticmethod
def _refused_agentic_tool_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
"""
Collect the ids and names of the tool calls a safety rail just refused.
Callbacks hand back either a bare list of tool calls or a dict wrapping
that list under ``tool_calls``, and both the anthropic and responses
shapes carry an ``id`` (or ``call_id``) plus a ``name``.
"""
calls: Final = tool_calls.get("tool_calls") if isinstance(tool_calls, dict) else tool_calls
if not isinstance(calls, list):
return frozenset(), frozenset()
dict_calls: Final = (call for call in calls if isinstance(call, dict))
fields: Final = tuple((call.get("id"), call.get("call_id"), call.get("name")) for call in dict_calls)
ids: Final = frozenset(
value for call_id, caller_id, _ in fields for value in (call_id, caller_id) if isinstance(value, str)
)
names: Final = frozenset(name for _, _, name in fields if isinstance(name, str))
return ids, names
@staticmethod
def _is_refused_tool_use_block(block: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
"""
@ -5894,22 +5879,13 @@ class BaseLLMHTTPHandler:
share a name with a refused one. The name is only consulted for tool call
shapes that arrive without an id.
"""
if not isinstance(block, dict) or block.get("type") != "tool_use":
return False
block_id: Final = block.get("id")
if isinstance(block_id, str) and refused_ids:
return block_id in refused_ids
return block.get("name") in refused_names
return is_refused_item(block, "tool_use", "id", refused_ids, refused_names)
@staticmethod
def _can_replace_turn_with_terminal_response(stream: bool, api_surface: str) -> bool:
"""
Whether a refused rerun can still be answered with a finalized turn.
Only the anthropic messages surface can. The responses surface carries a
pydantic model the finalizer does not rewrite, so it keeps raising, which
is what every surface did before this path learned to end the turn.
The messages and responses call sites pass ``stream=False``, because
interception converts an intercepted stream to non-streaming before the
loop runs and rebuilds the SSE stream from the finalized turn
@ -5918,7 +5894,7 @@ class BaseLLMHTTPHandler:
finalized turn would reach the client as a second message rather than
as a replacement.
"""
return not stream and api_surface == "anthropic_messages"
return not stream and api_surface in ("anthropic_messages", "responses")
@staticmethod
def _finalize_refused_agentic_response(response: object, tool_calls: object) -> object:
@ -5935,10 +5911,12 @@ class BaseLLMHTTPHandler:
one is still in the response the stop reason stays ``tool_use`` so the
client knows to answer it.
"""
if isinstance(response, ResponsesAPIResponse):
return finalize_refused_responses_turn(response, tool_calls)
if not isinstance(response, dict):
return response
refused_ids, refused_names = BaseLLMHTTPHandler._refused_agentic_tool_identifiers(tool_calls)
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
finalized: Final = dict(response)
content: Final = finalized.get("content")
if isinstance(content, list):

View file

@ -646,9 +646,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
# Get custom_llm_provider from litellm_params
custom_llm_provider: Final = litellm_params.get("custom_llm_provider", "openai")
for callback in callbacks:
if not isinstance(callback, CustomLogger):
continue
custom_loggers: Final = tuple(callback for callback in callbacks if isinstance(callback, CustomLogger))
for callback in custom_loggers:
try:
should_run, tool_calls = await callback.async_should_run_chat_completion_agentic_loop(
response=response,

View file

@ -31,6 +31,7 @@ from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.secret_managers.main import get_secret
from litellm.types.llms.openai import ResponsesAPIResponse
from litellm.types.integrations.custom_logger import (
AgenticLoopPlan,
AgenticLoopRequestPatch,
@ -99,6 +100,29 @@ def _tool_use_names(response: dict) -> list[str]:
return [block.get("name") for block in response["content"] if block.get("type") == "tool_use"]
def _responses_turn(*function_calls: tuple[str, str]) -> ResponsesAPIResponse:
return ResponsesAPIResponse(
id="resp_1",
created_at=0,
status="completed",
model="claude-sonnet-4-5",
object="response",
output=[
{
"type": "message",
"id": "msg_1",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "Let me check one more source.", "annotations": []}],
},
*(
{"type": "function_call", "id": f"fc_{call_id}", "call_id": call_id, "name": name, "arguments": "{}"}
for call_id, name in function_calls
),
],
)
class _InterceptingCallback(CustomLogger):
"""
Stands in for the websearch interceptor: asks for another loop whenever the
@ -152,6 +176,18 @@ class _InterceptingCallback(CustomLogger):
return response
class _ResponsesInterceptingCallback(_InterceptingCallback):
async def async_should_run_agentic_loop(
self, response, model, messages, tools, stream, custom_llm_provider, kwargs
):
calls = [
{"id": item.call_id, "call_id": item.call_id, "name": item.name}
for item in response.output
if item.type == "function_call" and item.name == INTERNAL_TOOL_NAME
]
return bool(calls), {"tool_calls": calls}
def _logging_obj(callback: CustomLogger, converted_stream: bool = False) -> MagicMock:
logging_obj = MagicMock()
logging_obj.model_call_details = {"websearch_interception_converted_stream": converted_stream}
@ -347,6 +383,23 @@ class TestCappedLoopReturnsTerminalResponse:
assert _tool_use_names(result) == []
assert result["stop_reason"] == "end_turn"
def test_a_name_only_refusal_keeps_the_native_search_blocks_sharing_that_name(self):
"""
The gate also picks up a bare web_search tool_use, and the native
server_tool_use blocks already in the turn carry that same name. Only the
tool_use goes, so each web_search_tool_result keeps the block it answers.
"""
response = _response_asking_for_another_search()
response["content"][-1] = {"type": "tool_use", "name": "web_search", "input": {}}
result = BaseLLMHTTPHandler._finalize_refused_agentic_response(
response=response,
tool_calls={"tool_calls": [{"name": "web_search", "input": {}}]},
)
assert _block_types(result) == ["server_tool_use", "web_search_tool_result", "text"]
assert result["stop_reason"] == "end_turn"
@pytest.mark.asyncio
async def test_streaming_caller_is_left_to_its_existing_behavior(self):
"""
@ -365,19 +418,39 @@ class TestCappedLoopReturnsTerminalResponse:
assert self.callback.plan_calls == 0
@pytest.mark.asyncio
async def test_responses_surface_is_left_to_its_existing_behavior(self):
async def test_responses_surface_drops_the_refused_function_call_and_keeps_the_answer(self):
response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_client", "lookup"))
callback = _ResponsesInterceptingCallback()
result = await _run_hooks(
self.handler,
callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
response=response,
api_surface="responses",
)
assert isinstance(result, ResponsesAPIResponse)
assert [(item.type, getattr(item, "name", None)) for item in result.output] == [
("message", None),
("function_call", "lookup"),
]
assert callback.plan_calls == 0
def test_responses_refusal_by_call_id_alone_keeps_the_calls_it_did_not_list(self):
"""
The responses surface carries a pydantic model rather than the anthropic
dict this finalizer rewrites, so it keeps raising instead of being handed
a response that was never actually finalized.
The code interpreter gate hands over call_id and name with no id, so the
call_id is what scopes the refusal there, the same way the id does on
/v1/messages.
"""
with pytest.raises(AgenticLoopSafetyError):
await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
api_surface="responses",
)
response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_unlisted", INTERNAL_TOOL_NAME))
result = BaseLLMHTTPHandler._finalize_refused_agentic_response(
response=response,
tool_calls={"tool_calls": [{"call_id": "call_search", "name": INTERNAL_TOOL_NAME}]},
)
assert [getattr(item, "call_id", None) for item in result.output] == [None, "call_unlisted"]
@pytest.mark.asyncio
async def test_non_dict_response_is_returned_untouched(self):

View file

@ -21,6 +21,7 @@ they stay out of the body even without it.
"""
import datetime
import json
from typing import Any, Dict, Final, List, Optional, Tuple
from unittest.mock import AsyncMock, MagicMock, patch
@ -43,6 +44,7 @@ from litellm.types.integrations.custom_logger import (
WEBSEARCH_INTERCEPTION_PREFIX,
AgenticLoopPlan,
AgenticLoopRequestPatch,
AgenticLoopSafetyError,
)
from litellm.types.utils import (
ChatCompletionMessageToolCall,
@ -398,72 +400,97 @@ async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_requ
assert call_kwargs["api_base"] == "https://a"
_REFUSED_GATE_TOOL_CALLS: Final = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]}
_REFUSING_RAILS: Final = (
{"_agentic_loop_depth": 3, "max_agentic_loops": 3},
{
"_agentic_loop_depth": 0,
"max_agentic_loops": 3,
"_agentic_loop_fingerprints": [json.dumps(_REFUSED_GATE_TOOL_CALLS, sort_keys=True, default=str)],
},
)
def _tool_call(call_id: str, name: str) -> ChatCompletionMessageToolCall:
return ChatCompletionMessageToolCall(id=call_id, type="function", function=Function(name=name, arguments="{}"))
async def _run_refused_rerun(
response: ModelResponse,
rail_kwargs: Dict[str, object],
stream: bool = False,
gate_tool_calls: Optional[Dict[str, object]] = _REFUSED_GATE_TOOL_CALLS,
) -> Any:
litellm.callbacks = [_GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=True), tool_calls=gate_tool_calls)]
acompletion_mock = AsyncMock()
with patch.object(litellm, "acompletion", acompletion_mock):
try:
return await maybe_run_chat_completion_agentic_loop(
response=response,
model="gpt-4o-mini",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
kwargs=dict(rail_kwargs),
logging_obj=_LoggingStub(),
custom_llm_provider="openai",
stream=stream,
)
finally:
acompletion_mock.assert_not_awaited()
@pytest.mark.asyncio
async def test_dispatcher_raises_when_depth_reaches_max_agentic_loops(
restore_callbacks,
@pytest.mark.parametrize("rail_kwargs", _REFUSING_RAILS, ids=["max_agentic_loops", "repeated_fingerprint"])
async def test_refused_rerun_ends_the_turn_without_the_internal_tool_call(
restore_callbacks, rail_kwargs: Dict[str, object]
):
"""depth >= max_agentic_loops -> ValueError mentioning max_agentic_loops,
before any follow-up call is attempted."""
logger = _GateOnlyLogger(
plan=AgenticLoopPlan(run_agentic_loop=True),
tool_calls={"tool_calls": [{"id": "call_abc"}]},
)
litellm.callbacks = [logger]
result = await _run_refused_rerun(_tool_call_model_response(), rail_kwargs)
acompletion_mock = AsyncMock()
with patch.object(litellm, "acompletion", acompletion_mock):
with pytest.raises(ValueError, match="max_agentic_loops"):
await maybe_run_chat_completion_agentic_loop(
response=_tool_call_model_response(),
model="gpt-4o-mini",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
logging_obj=_LoggingStub(),
custom_llm_provider="openai",
stream=False,
)
acompletion_mock.assert_not_awaited()
assert isinstance(result, ModelResponse)
assert not result.choices[0].message.tool_calls
assert result.choices[0].finish_reason == "stop"
@pytest.mark.asyncio
async def test_dispatcher_raises_on_repeated_tool_call_fingerprint(restore_callbacks):
"""A tool_calls fingerprint already present in _agentic_loop_fingerprints ->
ValueError about the repeated fingerprint (cycle guard), with no follow-up
call."""
import json
# The dispatcher fingerprints the whole value the gate returns as its second
# tuple element, so the seeded fingerprint must mirror that dict exactly.
gate_tool_calls = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]}
fingerprint = json.dumps(gate_tool_calls, sort_keys=True, default=str)
logger = _GateOnlyLogger(
plan=AgenticLoopPlan(run_agentic_loop=True),
tool_calls=gate_tool_calls,
)
litellm.callbacks = [logger]
acompletion_mock = AsyncMock()
with patch.object(litellm, "acompletion", acompletion_mock):
with pytest.raises(ValueError, match="fingerprint"):
await maybe_run_chat_completion_agentic_loop(
response=_tool_call_model_response(),
model="gpt-4o-mini",
messages=[{"role": "user", "content": "hi"}],
optional_params={},
kwargs={
"_agentic_loop_depth": 0,
"max_agentic_loops": 3,
"_agentic_loop_fingerprints": [fingerprint],
},
logging_obj=_LoggingStub(),
custom_llm_provider="openai",
stream=False,
async def test_refused_rerun_keeps_the_clients_own_tool_call(restore_callbacks):
response = ModelResponse(
choices=[
Choices(
finish_reason="tool_calls",
message=Message(
role="assistant",
content="Running it.",
tool_calls=[_tool_call("call_abc", "litellm_code_execution"), _tool_call("call_client", "lookup")],
),
)
]
)
acompletion_mock.assert_not_awaited()
result = await _run_refused_rerun(response, _REFUSING_RAILS[0])
assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"]
assert result.choices[0].finish_reason == "tool_calls"
assert result.choices[0].message.content == "Running it."
@pytest.mark.asyncio
@pytest.mark.parametrize(
"response, gate_tool_calls",
[(_plain_model_response(), _REFUSED_GATE_TOOL_CALLS), (_tool_call_model_response(), None)],
ids=["turn_has_no_tool_calls", "gate_names_no_tool_call"],
)
async def test_refused_rerun_returns_a_turn_it_cannot_match_unchanged(
restore_callbacks, response: ModelResponse, gate_tool_calls: Optional[Dict[str, object]]
):
result = await _run_refused_rerun(response, _REFUSING_RAILS[0], gate_tool_calls=gate_tool_calls)
assert result.choices[0].model_dump() == response.choices[0].model_dump()
@pytest.mark.asyncio
async def test_refused_rerun_on_a_live_stream_still_raises(restore_callbacks):
with pytest.raises(AgenticLoopSafetyError, match="max_agentic_loops"):
await _run_refused_rerun(_tool_call_model_response(), _REFUSING_RAILS[0], stream=True)
@pytest.mark.asyncio

View file

@ -256,6 +256,84 @@ class _FailingFollowUpChatLoop(CustomLogger):
raise litellm.RateLimitError(message="follow-up rate limited", llm_provider="openai", model=model)
class _RaisingAgenticGate(CustomLogger):
async def async_should_run_chat_completion_agentic_loop(
self,
response: object,
model: str,
messages: list[dict],
tools: list[dict] | None,
stream: bool,
custom_llm_provider: str,
kwargs: dict,
) -> tuple[bool, dict]:
raise ValueError("gate bug")
class _AnsweringChatLoop(_FailingFollowUpChatLoop):
async def async_run_chat_completion_agentic_loop(
self,
tools: dict,
model: str,
messages: list[dict],
response: object,
optional_params: dict,
logging_obj: object,
stream: bool,
kwargs: dict,
) -> object:
self.runs += 1
return litellm.ModelResponse(
choices=[
{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "after the search"}}
]
)
def _respond_with_a_search_tool_call(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json={
"id": "chatcmpl-search",
"object": "chat.completion",
"created": 0,
"model": "gpt-5.6",
"choices": [
{
"index": 0,
"finish_reason": "tool_calls",
"message": {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {"name": "litellm_web_search", "arguments": '{"query": "weather"}'},
}
],
},
}
],
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
},
)
async def _acompletion_through(http_client: httpx.AsyncClient) -> object:
return await asyncio.wait_for(
litellm.acompletion(
model="openai/gpt-5.6",
api_key="transport-only",
client=AsyncOpenAI(api_key="transport-only", http_client=http_client),
messages=[{"role": "user", "content": "search the web for the weather"}],
num_retries=0,
max_retries=0,
),
timeout=10,
)
@pytest.mark.asyncio
async def test_acompletion_surfaces_a_failed_agentic_follow_up_instead_of_the_first_turn(
monkeypatch: pytest.MonkeyPatch,
@ -263,51 +341,25 @@ async def test_acompletion_surfaces_a_failed_agentic_follow_up_instead_of_the_fi
loop: Final = _FailingFollowUpChatLoop()
monkeypatch.setattr(litellm, "callbacks", [loop])
def respond(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
json={
"id": "chatcmpl-search",
"object": "chat.completion",
"created": 0,
"model": "gpt-5.6",
"choices": [
{
"index": 0,
"finish_reason": "tool_calls",
"message": {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call-1",
"type": "function",
"function": {"name": "litellm_web_search", "arguments": '{"query": "weather"}'},
}
],
},
}
],
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
},
)
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client:
async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client:
with pytest.raises(litellm.RateLimitError, match="follow-up rate limited"):
await asyncio.wait_for(
litellm.acompletion(
model="openai/gpt-5.6",
api_key="transport-only",
client=AsyncOpenAI(api_key="transport-only", http_client=http_client),
messages=[{"role": "user", "content": "search the web for the weather"}],
num_retries=0,
max_retries=0,
),
timeout=10,
)
await _acompletion_through(http_client)
assert loop.runs == 1
@pytest.mark.asyncio
async def test_acompletion_skips_a_failing_agentic_gate_and_runs_the_next_hook(monkeypatch: pytest.MonkeyPatch):
loop: Final = _AnsweringChatLoop()
monkeypatch.setattr(litellm, "callbacks", [_RaisingAgenticGate(), loop])
async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client:
response = await _acompletion_through(http_client)
assert loop.runs == 1
assert response.choices[0].message.content == "after the search"
_REASONING_TOOL_CALL_REPLY: Final = {
"id": "chatcmpl-fake-stream",
"object": "chat.completion",