mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(agentic): end a capped agentic turn on chat and responses instead of a retried 500
This commit is contained in:
parent
b3849b29de
commit
e9ce495628
7 changed files with 390 additions and 154 deletions
72
litellm/litellm_core_utils/agentic_loop_refusal.py
Normal file
72
litellm/litellm_core_utils/agentic_loop_refusal.py
Normal file
|
|
@ -0,0 +1,72 @@
|
|||
from collections.abc import Iterable
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import message_field, parts_of
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.utils import Choices, ModelResponse
|
||||
|
||||
|
||||
def _strings(values: Iterable[object]) -> frozenset[str]:
|
||||
return frozenset(value for value in values if isinstance(value, str))
|
||||
|
||||
|
||||
def refused_tool_call_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
|
||||
wrapped: Final = message_field(tool_calls, "tool_calls")
|
||||
calls: Final = parts_of(tool_calls if wrapped is None else wrapped)
|
||||
ids: Final = _strings(
|
||||
chain.from_iterable((message_field(call, "id"), message_field(call, "call_id")) for call in calls)
|
||||
)
|
||||
names: Final = _strings(message_field(call, "name") for call in calls)
|
||||
return ids, names
|
||||
|
||||
|
||||
def is_refused_call(call_id: object, name: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
|
||||
if isinstance(call_id, str) and refused_ids:
|
||||
return call_id in refused_ids
|
||||
return name in refused_names
|
||||
|
||||
|
||||
def is_refused_item(
|
||||
item: object, item_type: str, id_field: str, refused_ids: frozenset[str], refused_names: frozenset[str]
|
||||
) -> bool:
|
||||
if message_field(item, "type") != item_type:
|
||||
return False
|
||||
return is_refused_call(message_field(item, id_field), message_field(item, "name"), refused_ids, refused_names)
|
||||
|
||||
|
||||
def finalize_refused_responses_turn(response: ResponsesAPIResponse, tool_calls: object) -> ResponsesAPIResponse:
|
||||
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
|
||||
kept: Final = [
|
||||
item
|
||||
for item in parts_of(message_field(response, "output"))
|
||||
if not is_refused_item(item, "function_call", "call_id", refused_ids, refused_names)
|
||||
]
|
||||
return response.model_copy(update={"output": kept})
|
||||
|
||||
|
||||
def _is_refused_chat_tool_call(call: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
|
||||
name: Final = message_field(message_field(call, "function"), "name")
|
||||
return is_refused_call(message_field(call, "id"), name, refused_ids, refused_names)
|
||||
|
||||
|
||||
def _finalized_choice(choice: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> object:
|
||||
if not isinstance(choice, Choices) or not choice.message.tool_calls:
|
||||
return choice
|
||||
kept: Final = [
|
||||
call for call in choice.message.tool_calls if not _is_refused_chat_tool_call(call, refused_ids, refused_names)
|
||||
]
|
||||
finish_reason: Final = "stop" if not kept and choice.finish_reason == "tool_calls" else choice.finish_reason
|
||||
return choice.model_copy(
|
||||
update={
|
||||
"message": choice.message.model_copy(update={"tool_calls": kept or None}),
|
||||
"finish_reason": finish_reason,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def finalize_refused_chat_turn(response: ModelResponse, tool_calls: object) -> ModelResponse:
|
||||
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
|
||||
return response.model_copy(
|
||||
update={"choices": [_finalized_choice(choice, refused_ids, refused_names) for choice in response.choices]}
|
||||
)
|
||||
|
|
@ -11,6 +11,7 @@ from pydantic import TypeAdapter, ValidationError
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
|
||||
from litellm.litellm_core_utils.agentic_loop_refusal import finalize_refused_chat_turn
|
||||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
DEFAULT_MAX_AGENTIC_LOOPS,
|
||||
validated_max_agentic_loops,
|
||||
|
|
@ -22,6 +23,7 @@ from litellm.types.integrations.custom_logger import (
|
|||
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
|
||||
AgenticLoopPlan,
|
||||
AgenticLoopRequestPatch,
|
||||
AgenticLoopSafetyError,
|
||||
converted_stream_requested,
|
||||
is_interception_internal_key,
|
||||
stashed_stream_options,
|
||||
|
|
@ -101,9 +103,9 @@ def _check_agentic_loop_safety(
|
|||
) -> str:
|
||||
fingerprint: Final = _fingerprint_tools(tool_calls)
|
||||
if fingerprint in fingerprints:
|
||||
raise ValueError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
|
||||
raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
|
||||
if depth >= max_loops:
|
||||
raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
|
||||
raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
|
||||
return fingerprint
|
||||
|
||||
|
||||
|
|
@ -132,6 +134,26 @@ def _converted_stream_replay(
|
|||
)
|
||||
|
||||
|
||||
def _end_refused_turn(
|
||||
finalized: ModelResponse,
|
||||
*,
|
||||
kwargs: Mapping[str, object],
|
||||
depth: int,
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
logging_obj: object,
|
||||
) -> ModelResponse | CustomStreamWrapper:
|
||||
replay: Final = _converted_stream_replay(
|
||||
finalized,
|
||||
kwargs=kwargs,
|
||||
depth=depth,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
return finalized if replay is None else replay
|
||||
|
||||
|
||||
def _with_agentic_loop_metadata(kwargs_for_followup: Mapping[str, object]) -> Mapping[str, object]:
|
||||
metadata: Final = kwargs_for_followup.get("litellm_metadata")
|
||||
return MappingProxyType(
|
||||
|
|
@ -296,13 +318,26 @@ async def maybe_run_chat_completion_agentic_loop(
|
|||
if not should_run:
|
||||
continue
|
||||
|
||||
fingerprint = _check_agentic_loop_safety(
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model=model,
|
||||
)
|
||||
try:
|
||||
fingerprint = _check_agentic_loop_safety(
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model=model,
|
||||
)
|
||||
except AgenticLoopSafetyError as e:
|
||||
if stream:
|
||||
raise
|
||||
verbose_logger.warning("LiteLLM.AgenticLoopRefused: ending turn [model=%s]: %s", model, str(e))
|
||||
return _end_refused_turn(
|
||||
finalize_refused_chat_turn(response, tool_calls),
|
||||
kwargs=kwargs,
|
||||
depth=depth,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
logging_obj=logging_obj,
|
||||
)
|
||||
|
||||
if not _build_plan_overridden(callback):
|
||||
return await callback.async_run_agentic_loop(
|
||||
|
|
|
|||
|
|
@ -41,6 +41,11 @@ from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, up
|
|||
from litellm.constants import MAX_FILE_LIST_LIMIT, REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
from litellm.files.types import FileContentStreamingResult
|
||||
from litellm.litellm_core_utils.agentic_followup_kwargs import build_agentic_followup_kwargs
|
||||
from litellm.litellm_core_utils.agentic_loop_refusal import (
|
||||
finalize_refused_responses_turn,
|
||||
is_refused_item,
|
||||
refused_tool_call_identifiers,
|
||||
)
|
||||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
DEFAULT_MAX_AGENTIC_LOOPS,
|
||||
validated_max_agentic_loops,
|
||||
|
|
@ -5864,26 +5869,6 @@ class BaseLLMHTTPHandler:
|
|||
except Exception:
|
||||
return str(tools)
|
||||
|
||||
@staticmethod
|
||||
def _refused_agentic_tool_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
|
||||
"""
|
||||
Collect the ids and names of the tool calls a safety rail just refused.
|
||||
|
||||
Callbacks hand back either a bare list of tool calls or a dict wrapping
|
||||
that list under ``tool_calls``, and both the anthropic and responses
|
||||
shapes carry an ``id`` (or ``call_id``) plus a ``name``.
|
||||
"""
|
||||
calls: Final = tool_calls.get("tool_calls") if isinstance(tool_calls, dict) else tool_calls
|
||||
if not isinstance(calls, list):
|
||||
return frozenset(), frozenset()
|
||||
dict_calls: Final = (call for call in calls if isinstance(call, dict))
|
||||
fields: Final = tuple((call.get("id"), call.get("call_id"), call.get("name")) for call in dict_calls)
|
||||
ids: Final = frozenset(
|
||||
value for call_id, caller_id, _ in fields for value in (call_id, caller_id) if isinstance(value, str)
|
||||
)
|
||||
names: Final = frozenset(name for _, _, name in fields if isinstance(name, str))
|
||||
return ids, names
|
||||
|
||||
@staticmethod
|
||||
def _is_refused_tool_use_block(block: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
|
||||
"""
|
||||
|
|
@ -5894,22 +5879,13 @@ class BaseLLMHTTPHandler:
|
|||
share a name with a refused one. The name is only consulted for tool call
|
||||
shapes that arrive without an id.
|
||||
"""
|
||||
if not isinstance(block, dict) or block.get("type") != "tool_use":
|
||||
return False
|
||||
block_id: Final = block.get("id")
|
||||
if isinstance(block_id, str) and refused_ids:
|
||||
return block_id in refused_ids
|
||||
return block.get("name") in refused_names
|
||||
return is_refused_item(block, "tool_use", "id", refused_ids, refused_names)
|
||||
|
||||
@staticmethod
|
||||
def _can_replace_turn_with_terminal_response(stream: bool, api_surface: str) -> bool:
|
||||
"""
|
||||
Whether a refused rerun can still be answered with a finalized turn.
|
||||
|
||||
Only the anthropic messages surface can. The responses surface carries a
|
||||
pydantic model the finalizer does not rewrite, so it keeps raising, which
|
||||
is what every surface did before this path learned to end the turn.
|
||||
|
||||
The messages and responses call sites pass ``stream=False``, because
|
||||
interception converts an intercepted stream to non-streaming before the
|
||||
loop runs and rebuilds the SSE stream from the finalized turn
|
||||
|
|
@ -5918,7 +5894,7 @@ class BaseLLMHTTPHandler:
|
|||
finalized turn would reach the client as a second message rather than
|
||||
as a replacement.
|
||||
"""
|
||||
return not stream and api_surface == "anthropic_messages"
|
||||
return not stream and api_surface in ("anthropic_messages", "responses")
|
||||
|
||||
@staticmethod
|
||||
def _finalize_refused_agentic_response(response: object, tool_calls: object) -> object:
|
||||
|
|
@ -5935,10 +5911,12 @@ class BaseLLMHTTPHandler:
|
|||
one is still in the response the stop reason stays ``tool_use`` so the
|
||||
client knows to answer it.
|
||||
"""
|
||||
if isinstance(response, ResponsesAPIResponse):
|
||||
return finalize_refused_responses_turn(response, tool_calls)
|
||||
if not isinstance(response, dict):
|
||||
return response
|
||||
|
||||
refused_ids, refused_names = BaseLLMHTTPHandler._refused_agentic_tool_identifiers(tool_calls)
|
||||
refused_ids, refused_names = refused_tool_call_identifiers(tool_calls)
|
||||
finalized: Final = dict(response)
|
||||
content: Final = finalized.get("content")
|
||||
if isinstance(content, list):
|
||||
|
|
|
|||
|
|
@ -646,9 +646,8 @@ class OpenAIChatCompletion(BaseLLM, BaseOpenAILLM):
|
|||
# Get custom_llm_provider from litellm_params
|
||||
custom_llm_provider: Final = litellm_params.get("custom_llm_provider", "openai")
|
||||
|
||||
for callback in callbacks:
|
||||
if not isinstance(callback, CustomLogger):
|
||||
continue
|
||||
custom_loggers: Final = tuple(callback for callback in callbacks if isinstance(callback, CustomLogger))
|
||||
for callback in custom_loggers:
|
||||
try:
|
||||
should_run, tool_calls = await callback.async_should_run_chat_completion_agentic_loop(
|
||||
response=response,
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ from litellm.llms.anthropic.messages.native.fake_stream_iterator import (
|
|||
from litellm.llms.base_llm.search.transformation import SearchResponse, SearchResult
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.secret_managers.main import get_secret
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
AgenticLoopPlan,
|
||||
AgenticLoopRequestPatch,
|
||||
|
|
@ -99,6 +100,29 @@ def _tool_use_names(response: dict) -> list[str]:
|
|||
return [block.get("name") for block in response["content"] if block.get("type") == "tool_use"]
|
||||
|
||||
|
||||
def _responses_turn(*function_calls: tuple[str, str]) -> ResponsesAPIResponse:
|
||||
return ResponsesAPIResponse(
|
||||
id="resp_1",
|
||||
created_at=0,
|
||||
status="completed",
|
||||
model="claude-sonnet-4-5",
|
||||
object="response",
|
||||
output=[
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_1",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "Let me check one more source.", "annotations": []}],
|
||||
},
|
||||
*(
|
||||
{"type": "function_call", "id": f"fc_{call_id}", "call_id": call_id, "name": name, "arguments": "{}"}
|
||||
for call_id, name in function_calls
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
class _InterceptingCallback(CustomLogger):
|
||||
"""
|
||||
Stands in for the websearch interceptor: asks for another loop whenever the
|
||||
|
|
@ -152,6 +176,18 @@ class _InterceptingCallback(CustomLogger):
|
|||
return response
|
||||
|
||||
|
||||
class _ResponsesInterceptingCallback(_InterceptingCallback):
|
||||
async def async_should_run_agentic_loop(
|
||||
self, response, model, messages, tools, stream, custom_llm_provider, kwargs
|
||||
):
|
||||
calls = [
|
||||
{"id": item.call_id, "call_id": item.call_id, "name": item.name}
|
||||
for item in response.output
|
||||
if item.type == "function_call" and item.name == INTERNAL_TOOL_NAME
|
||||
]
|
||||
return bool(calls), {"tool_calls": calls}
|
||||
|
||||
|
||||
def _logging_obj(callback: CustomLogger, converted_stream: bool = False) -> MagicMock:
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.model_call_details = {"websearch_interception_converted_stream": converted_stream}
|
||||
|
|
@ -347,6 +383,23 @@ class TestCappedLoopReturnsTerminalResponse:
|
|||
assert _tool_use_names(result) == []
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
|
||||
def test_a_name_only_refusal_keeps_the_native_search_blocks_sharing_that_name(self):
|
||||
"""
|
||||
The gate also picks up a bare web_search tool_use, and the native
|
||||
server_tool_use blocks already in the turn carry that same name. Only the
|
||||
tool_use goes, so each web_search_tool_result keeps the block it answers.
|
||||
"""
|
||||
response = _response_asking_for_another_search()
|
||||
response["content"][-1] = {"type": "tool_use", "name": "web_search", "input": {}}
|
||||
|
||||
result = BaseLLMHTTPHandler._finalize_refused_agentic_response(
|
||||
response=response,
|
||||
tool_calls={"tool_calls": [{"name": "web_search", "input": {}}]},
|
||||
)
|
||||
|
||||
assert _block_types(result) == ["server_tool_use", "web_search_tool_result", "text"]
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_caller_is_left_to_its_existing_behavior(self):
|
||||
"""
|
||||
|
|
@ -365,19 +418,39 @@ class TestCappedLoopReturnsTerminalResponse:
|
|||
assert self.callback.plan_calls == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_surface_is_left_to_its_existing_behavior(self):
|
||||
async def test_responses_surface_drops_the_refused_function_call_and_keeps_the_answer(self):
|
||||
response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_client", "lookup"))
|
||||
callback = _ResponsesInterceptingCallback()
|
||||
|
||||
result = await _run_hooks(
|
||||
self.handler,
|
||||
callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
response=response,
|
||||
api_surface="responses",
|
||||
)
|
||||
|
||||
assert isinstance(result, ResponsesAPIResponse)
|
||||
assert [(item.type, getattr(item, "name", None)) for item in result.output] == [
|
||||
("message", None),
|
||||
("function_call", "lookup"),
|
||||
]
|
||||
assert callback.plan_calls == 0
|
||||
|
||||
def test_responses_refusal_by_call_id_alone_keeps_the_calls_it_did_not_list(self):
|
||||
"""
|
||||
The responses surface carries a pydantic model rather than the anthropic
|
||||
dict this finalizer rewrites, so it keeps raising instead of being handed
|
||||
a response that was never actually finalized.
|
||||
The code interpreter gate hands over call_id and name with no id, so the
|
||||
call_id is what scopes the refusal there, the same way the id does on
|
||||
/v1/messages.
|
||||
"""
|
||||
with pytest.raises(AgenticLoopSafetyError):
|
||||
await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
api_surface="responses",
|
||||
)
|
||||
response = _responses_turn(("call_search", INTERNAL_TOOL_NAME), ("call_unlisted", INTERNAL_TOOL_NAME))
|
||||
|
||||
result = BaseLLMHTTPHandler._finalize_refused_agentic_response(
|
||||
response=response,
|
||||
tool_calls={"tool_calls": [{"call_id": "call_search", "name": INTERNAL_TOOL_NAME}]},
|
||||
)
|
||||
|
||||
assert [getattr(item, "call_id", None) for item in result.output] == [None, "call_unlisted"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_dict_response_is_returned_untouched(self):
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ they stay out of the body even without it.
|
|||
"""
|
||||
|
||||
import datetime
|
||||
import json
|
||||
from typing import Any, Dict, Final, List, Optional, Tuple
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
|
|
@ -43,6 +44,7 @@ from litellm.types.integrations.custom_logger import (
|
|||
WEBSEARCH_INTERCEPTION_PREFIX,
|
||||
AgenticLoopPlan,
|
||||
AgenticLoopRequestPatch,
|
||||
AgenticLoopSafetyError,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
ChatCompletionMessageToolCall,
|
||||
|
|
@ -398,72 +400,97 @@ async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_requ
|
|||
assert call_kwargs["api_base"] == "https://a"
|
||||
|
||||
|
||||
_REFUSED_GATE_TOOL_CALLS: Final = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]}
|
||||
_REFUSING_RAILS: Final = (
|
||||
{"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
{
|
||||
"_agentic_loop_depth": 0,
|
||||
"max_agentic_loops": 3,
|
||||
"_agentic_loop_fingerprints": [json.dumps(_REFUSED_GATE_TOOL_CALLS, sort_keys=True, default=str)],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _tool_call(call_id: str, name: str) -> ChatCompletionMessageToolCall:
|
||||
return ChatCompletionMessageToolCall(id=call_id, type="function", function=Function(name=name, arguments="{}"))
|
||||
|
||||
|
||||
async def _run_refused_rerun(
|
||||
response: ModelResponse,
|
||||
rail_kwargs: Dict[str, object],
|
||||
stream: bool = False,
|
||||
gate_tool_calls: Optional[Dict[str, object]] = _REFUSED_GATE_TOOL_CALLS,
|
||||
) -> Any:
|
||||
litellm.callbacks = [_GateOnlyLogger(plan=AgenticLoopPlan(run_agentic_loop=True), tool_calls=gate_tool_calls)]
|
||||
acompletion_mock = AsyncMock()
|
||||
with patch.object(litellm, "acompletion", acompletion_mock):
|
||||
try:
|
||||
return await maybe_run_chat_completion_agentic_loop(
|
||||
response=response,
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
kwargs=dict(rail_kwargs),
|
||||
logging_obj=_LoggingStub(),
|
||||
custom_llm_provider="openai",
|
||||
stream=stream,
|
||||
)
|
||||
finally:
|
||||
acompletion_mock.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatcher_raises_when_depth_reaches_max_agentic_loops(
|
||||
restore_callbacks,
|
||||
@pytest.mark.parametrize("rail_kwargs", _REFUSING_RAILS, ids=["max_agentic_loops", "repeated_fingerprint"])
|
||||
async def test_refused_rerun_ends_the_turn_without_the_internal_tool_call(
|
||||
restore_callbacks, rail_kwargs: Dict[str, object]
|
||||
):
|
||||
"""depth >= max_agentic_loops -> ValueError mentioning max_agentic_loops,
|
||||
before any follow-up call is attempted."""
|
||||
logger = _GateOnlyLogger(
|
||||
plan=AgenticLoopPlan(run_agentic_loop=True),
|
||||
tool_calls={"tool_calls": [{"id": "call_abc"}]},
|
||||
)
|
||||
litellm.callbacks = [logger]
|
||||
result = await _run_refused_rerun(_tool_call_model_response(), rail_kwargs)
|
||||
|
||||
acompletion_mock = AsyncMock()
|
||||
with patch.object(litellm, "acompletion", acompletion_mock):
|
||||
with pytest.raises(ValueError, match="max_agentic_loops"):
|
||||
await maybe_run_chat_completion_agentic_loop(
|
||||
response=_tool_call_model_response(),
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
logging_obj=_LoggingStub(),
|
||||
custom_llm_provider="openai",
|
||||
stream=False,
|
||||
)
|
||||
|
||||
acompletion_mock.assert_not_awaited()
|
||||
assert isinstance(result, ModelResponse)
|
||||
assert not result.choices[0].message.tool_calls
|
||||
assert result.choices[0].finish_reason == "stop"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatcher_raises_on_repeated_tool_call_fingerprint(restore_callbacks):
|
||||
"""A tool_calls fingerprint already present in _agentic_loop_fingerprints ->
|
||||
ValueError about the repeated fingerprint (cycle guard), with no follow-up
|
||||
call."""
|
||||
import json
|
||||
|
||||
# The dispatcher fingerprints the whole value the gate returns as its second
|
||||
# tuple element, so the seeded fingerprint must mirror that dict exactly.
|
||||
gate_tool_calls = {"tool_calls": [{"id": "call_abc", "name": "litellm_code_execution"}]}
|
||||
fingerprint = json.dumps(gate_tool_calls, sort_keys=True, default=str)
|
||||
|
||||
logger = _GateOnlyLogger(
|
||||
plan=AgenticLoopPlan(run_agentic_loop=True),
|
||||
tool_calls=gate_tool_calls,
|
||||
)
|
||||
litellm.callbacks = [logger]
|
||||
|
||||
acompletion_mock = AsyncMock()
|
||||
with patch.object(litellm, "acompletion", acompletion_mock):
|
||||
with pytest.raises(ValueError, match="fingerprint"):
|
||||
await maybe_run_chat_completion_agentic_loop(
|
||||
response=_tool_call_model_response(),
|
||||
model="gpt-4o-mini",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={},
|
||||
kwargs={
|
||||
"_agentic_loop_depth": 0,
|
||||
"max_agentic_loops": 3,
|
||||
"_agentic_loop_fingerprints": [fingerprint],
|
||||
},
|
||||
logging_obj=_LoggingStub(),
|
||||
custom_llm_provider="openai",
|
||||
stream=False,
|
||||
async def test_refused_rerun_keeps_the_clients_own_tool_call(restore_callbacks):
|
||||
response = ModelResponse(
|
||||
choices=[
|
||||
Choices(
|
||||
finish_reason="tool_calls",
|
||||
message=Message(
|
||||
role="assistant",
|
||||
content="Running it.",
|
||||
tool_calls=[_tool_call("call_abc", "litellm_code_execution"), _tool_call("call_client", "lookup")],
|
||||
),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
acompletion_mock.assert_not_awaited()
|
||||
result = await _run_refused_rerun(response, _REFUSING_RAILS[0])
|
||||
|
||||
assert [call.id for call in result.choices[0].message.tool_calls] == ["call_client"]
|
||||
assert result.choices[0].finish_reason == "tool_calls"
|
||||
assert result.choices[0].message.content == "Running it."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"response, gate_tool_calls",
|
||||
[(_plain_model_response(), _REFUSED_GATE_TOOL_CALLS), (_tool_call_model_response(), None)],
|
||||
ids=["turn_has_no_tool_calls", "gate_names_no_tool_call"],
|
||||
)
|
||||
async def test_refused_rerun_returns_a_turn_it_cannot_match_unchanged(
|
||||
restore_callbacks, response: ModelResponse, gate_tool_calls: Optional[Dict[str, object]]
|
||||
):
|
||||
result = await _run_refused_rerun(response, _REFUSING_RAILS[0], gate_tool_calls=gate_tool_calls)
|
||||
|
||||
assert result.choices[0].model_dump() == response.choices[0].model_dump()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refused_rerun_on_a_live_stream_still_raises(restore_callbacks):
|
||||
with pytest.raises(AgenticLoopSafetyError, match="max_agentic_loops"):
|
||||
await _run_refused_rerun(_tool_call_model_response(), _REFUSING_RAILS[0], stream=True)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -256,6 +256,84 @@ class _FailingFollowUpChatLoop(CustomLogger):
|
|||
raise litellm.RateLimitError(message="follow-up rate limited", llm_provider="openai", model=model)
|
||||
|
||||
|
||||
class _RaisingAgenticGate(CustomLogger):
|
||||
async def async_should_run_chat_completion_agentic_loop(
|
||||
self,
|
||||
response: object,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
tools: list[dict] | None,
|
||||
stream: bool,
|
||||
custom_llm_provider: str,
|
||||
kwargs: dict,
|
||||
) -> tuple[bool, dict]:
|
||||
raise ValueError("gate bug")
|
||||
|
||||
|
||||
class _AnsweringChatLoop(_FailingFollowUpChatLoop):
|
||||
async def async_run_chat_completion_agentic_loop(
|
||||
self,
|
||||
tools: dict,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
response: object,
|
||||
optional_params: dict,
|
||||
logging_obj: object,
|
||||
stream: bool,
|
||||
kwargs: dict,
|
||||
) -> object:
|
||||
self.runs += 1
|
||||
return litellm.ModelResponse(
|
||||
choices=[
|
||||
{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "after the search"}}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _respond_with_a_search_tool_call(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-search",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gpt-5.6",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "tool_calls",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call-1",
|
||||
"type": "function",
|
||||
"function": {"name": "litellm_web_search", "arguments": '{"query": "weather"}'},
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def _acompletion_through(http_client: httpx.AsyncClient) -> object:
|
||||
return await asyncio.wait_for(
|
||||
litellm.acompletion(
|
||||
model="openai/gpt-5.6",
|
||||
api_key="transport-only",
|
||||
client=AsyncOpenAI(api_key="transport-only", http_client=http_client),
|
||||
messages=[{"role": "user", "content": "search the web for the weather"}],
|
||||
num_retries=0,
|
||||
max_retries=0,
|
||||
),
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_surfaces_a_failed_agentic_follow_up_instead_of_the_first_turn(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
|
|
@ -263,51 +341,25 @@ async def test_acompletion_surfaces_a_failed_agentic_follow_up_instead_of_the_fi
|
|||
loop: Final = _FailingFollowUpChatLoop()
|
||||
monkeypatch.setattr(litellm, "callbacks", [loop])
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"id": "chatcmpl-search",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": "gpt-5.6",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "tool_calls",
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call-1",
|
||||
"type": "function",
|
||||
"function": {"name": "litellm_web_search", "arguments": '{"query": "weather"}'},
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 3, "total_tokens": 8},
|
||||
},
|
||||
)
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as http_client:
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client:
|
||||
with pytest.raises(litellm.RateLimitError, match="follow-up rate limited"):
|
||||
await asyncio.wait_for(
|
||||
litellm.acompletion(
|
||||
model="openai/gpt-5.6",
|
||||
api_key="transport-only",
|
||||
client=AsyncOpenAI(api_key="transport-only", http_client=http_client),
|
||||
messages=[{"role": "user", "content": "search the web for the weather"}],
|
||||
num_retries=0,
|
||||
max_retries=0,
|
||||
),
|
||||
timeout=10,
|
||||
)
|
||||
await _acompletion_through(http_client)
|
||||
|
||||
assert loop.runs == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_acompletion_skips_a_failing_agentic_gate_and_runs_the_next_hook(monkeypatch: pytest.MonkeyPatch):
|
||||
loop: Final = _AnsweringChatLoop()
|
||||
monkeypatch.setattr(litellm, "callbacks", [_RaisingAgenticGate(), loop])
|
||||
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(_respond_with_a_search_tool_call)) as http_client:
|
||||
response = await _acompletion_through(http_client)
|
||||
|
||||
assert loop.runs == 1
|
||||
assert response.choices[0].message.content == "after the search"
|
||||
|
||||
|
||||
_REASONING_TOOL_CALL_REPLY: Final = {
|
||||
"id": "chatcmpl-fake-stream",
|
||||
"object": "chat.completion",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue