mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-12 23:01:41 +00:00
Merge pull request #37911 from BerriAI/litellm_fix_agentic_loop_cap_response
fix(websearch_interception): end the turn when the agentic loop hits its ceiling
This commit is contained in:
commit
abf99e37d6
11 changed files with 1144 additions and 14 deletions
|
|
@ -207,6 +207,59 @@ response = await litellm.messages.acreate(
|
|||
|
||||
---
|
||||
|
||||
## Loop Ceiling
|
||||
|
||||
One intercepted request can chain several follow-up model calls, since the model often searches again after
|
||||
reading the first set of results. `max_agentic_loops` caps how many of those follow-ups run, and it defaults
|
||||
to 3. LiteLLM also breaks the loop early when the model asks for the exact same tool call twice in a row.
|
||||
|
||||
Set the ceiling on the feature, which the interceptor applies to `/v1/messages` requests:
|
||||
|
||||
```yaml
|
||||
litellm_settings:
|
||||
websearch_interception_params:
|
||||
enabled_providers: ["bedrock"]
|
||||
max_agentic_loops: 5
|
||||
```
|
||||
|
||||
Or per deployment, which wins over the feature-level setting:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: claude-sonnet-4-5
|
||||
litellm_params:
|
||||
model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
|
||||
max_agentic_loops: 5
|
||||
```
|
||||
|
||||
Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that
|
||||
carries it is ignored and one request can never drive an unbounded number of upstream model calls.
|
||||
|
||||
Both places are validated at config load, and a value that is not an integer of at least 1 stops the proxy
|
||||
from starting rather than surfacing later. The per-deployment one is checked while the model list is read,
|
||||
not on `LiteLLM_Params`, because the proxy builds its router with `ignore_invalid_deployments=True` and a
|
||||
validator down there would drop the deployment silently instead of refusing to start.
|
||||
|
||||
When the ceiling is reached on a non-streaming `/v1/messages` request, the turn ends there and the client gets
|
||||
the last response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`.
|
||||
The client never declared that tool, so leaving the block in would hand it a tool call it has no way to answer.
|
||||
The answer can be less complete than it would have been with more loops, which is the tradeoff the ceiling
|
||||
buys. Where the refused call was the only block left, the turn comes back with no text in it at all.
|
||||
|
||||
Non-streaming is not a limitation on the client here, because a client that asked for a stream gets the same
|
||||
treatment. Interception converts an intercepted `stream=True` request to non-streaming before the loop runs and
|
||||
rebuilds the SSE stream from the finalized turn afterwards, so the ceiling is always reached on a response the
|
||||
client has not seen yet. `AgenticStreamingIterator` is the one caller that reaches the loop with its events
|
||||
already on the wire, and it keeps raising, because a finalized turn would arrive there as a second message
|
||||
rather than as a replacement.
|
||||
|
||||
Two other surfaces do not get that treatment yet. `/v1/responses` returns its own shape that the finalizer does
|
||||
not rewrite, so it still hands back the internal call. And `/v1/chat/completions` runs its own copy of these
|
||||
rails in `litellm_core_utils/chat_completion_agentic_loop.py`, which still raises rather than ending the turn.
|
||||
Both are tracked separately
|
||||
|
||||
---
|
||||
|
||||
## Streaming Support
|
||||
|
||||
WebSearch interception works transparently with both streaming and non-streaming requests.
|
||||
|
|
|
|||
|
|
@ -31,6 +31,9 @@ from litellm.integrations.websearch_interception.tools import (
|
|||
from litellm.integrations.websearch_interception.transformation import (
|
||||
WebSearchTransformation,
|
||||
)
|
||||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.llms.base_llm.search.transformation import SearchResponse
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
CHAT_COMPLETION_AGENTIC_SURFACE,
|
||||
|
|
@ -122,6 +125,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
self,
|
||||
enabled_providers: list[LlmProviders | str] | None = None,
|
||||
search_tool_name: str | None = None,
|
||||
max_agentic_loops: int | None = None,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
|
|
@ -131,6 +135,9 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
Default: None (all providers enabled)
|
||||
search_tool_name: Name of search tool configured in router's search_tools.
|
||||
If None, will attempt to use first available search tool.
|
||||
max_agentic_loops: How many follow-up model calls one intercepted request
|
||||
may chain before the loop is refused and the turn ends.
|
||||
If None, LiteLLM's default of 3 applies.
|
||||
"""
|
||||
super().__init__()
|
||||
# Convert enum values to strings for comparison
|
||||
|
|
@ -139,8 +146,16 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
else:
|
||||
self.enabled_providers = [p.value if isinstance(p, LlmProviders) else p for p in enabled_providers]
|
||||
self.search_tool_name = search_tool_name
|
||||
self.max_agentic_loops = self._validated_max_agentic_loops(max_agentic_loops)
|
||||
self._request_has_websearch = False # Track if current request has web search
|
||||
|
||||
@staticmethod
|
||||
def _validated_max_agentic_loops(max_agentic_loops: object) -> int | None:
|
||||
"""
|
||||
Reject loop ceilings the agentic loop cannot honor, at config load time.
|
||||
"""
|
||||
return validated_max_agentic_loops(max_agentic_loops, field="websearch_interception_params.max_agentic_loops")
|
||||
|
||||
async def try_short_circuit_search(
|
||||
self,
|
||||
model: str,
|
||||
|
|
@ -398,6 +413,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
websearch_interception_params:
|
||||
enabled_providers: ["bedrock"]
|
||||
search_tool_name: "my-perplexity-search"
|
||||
max_agentic_loops: 5
|
||||
|
||||
Usage:
|
||||
config = litellm_settings.get("websearch_interception_params", {})
|
||||
|
|
@ -406,6 +422,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
# Extract parameters from config
|
||||
enabled_providers_str: Final = config.get("enabled_providers", None)
|
||||
search_tool_name: Final = config.get("search_tool_name", None)
|
||||
max_agentic_loops: Final = config.get("max_agentic_loops", None)
|
||||
|
||||
# Convert string provider names to LlmProviders enum values
|
||||
enabled_providers: list[LlmProviders | str] | None = None
|
||||
|
|
@ -423,6 +440,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
return cls(
|
||||
enabled_providers=enabled_providers,
|
||||
search_tool_name=search_tool_name,
|
||||
max_agentic_loops=max_agentic_loops,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -493,6 +511,10 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
|
||||
verbose_logger.debug("WebSearchInterception: Pre-request hook triggered for provider=%s", custom_llm_provider)
|
||||
|
||||
deployment_max_agentic_loops: Final = kwargs.get("max_agentic_loops")
|
||||
if self.max_agentic_loops is not None and deployment_max_agentic_loops is None:
|
||||
kwargs["max_agentic_loops"] = self.max_agentic_loops # rebind-ok: this hook returns the kwargs it edits
|
||||
|
||||
# If the client sent an Anthropic-native web_search_* tool, mark the
|
||||
# request so the agentic loop emits native web_search_tool_result
|
||||
# blocks in the final response (for citations panels, etc.). The flag
|
||||
|
|
|
|||
59
litellm/litellm_core_utils/agentic_loop_settings.py
Normal file
59
litellm/litellm_core_utils/agentic_loop_settings.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
"""
|
||||
Shared validation for the agentic loop ceiling.
|
||||
|
||||
``max_agentic_loops`` can be set in two places, and the two disagreed about
|
||||
what a bad value means. The feature-level
|
||||
``litellm_settings.websearch_interception_params.max_agentic_loops`` was
|
||||
checked at config load, while a per-deployment
|
||||
``model_list[].litellm_params.max_agentic_loops`` was passed straight through
|
||||
to ``int(... or 3)``. That let a per-deployment ``0`` read as the default 3,
|
||||
turning the tightest ceiling into the loosest one, and let a per-deployment
|
||||
``"three"`` boot the proxy and then fail every request to that model.
|
||||
|
||||
Both settings now go through :func:`validated_max_agentic_loops`, which names
|
||||
the field it rejected so the error says which line of the config to fix.
|
||||
|
||||
Anything that spells a whole number is still accepted, because the old
|
||||
``int(... or 3)`` accepted those and a ceiling is routinely parameterized as
|
||||
``max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS``, which resolves to a
|
||||
string. Rejecting ``"5"`` would stop such a proxy from booting on upgrade.
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
|
||||
DEFAULT_MAX_AGENTIC_LOOPS: Final = 3
|
||||
|
||||
|
||||
def _as_whole_number(value: object) -> int | None:
|
||||
"""
|
||||
Return ``value`` as an int when it spells a whole number, else ``None``.
|
||||
|
||||
``bool`` is excluded explicitly because it is an ``int`` subclass, so
|
||||
``max_agentic_loops: true`` would otherwise be read as a ceiling of 1.
|
||||
"""
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
return int(value) if value.is_integer() else None
|
||||
if isinstance(value, str):
|
||||
try:
|
||||
return int(value.strip())
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def validated_max_agentic_loops(max_agentic_loops: object, field: str) -> int | None:
|
||||
"""
|
||||
Return ``max_agentic_loops`` as an int, or raise naming ``field``.
|
||||
"""
|
||||
if max_agentic_loops is None:
|
||||
return None
|
||||
ceiling: Final = _as_whole_number(max_agentic_loops)
|
||||
if ceiling is None:
|
||||
raise TypeError(f"{field} must be an integer, got {max_agentic_loops!r}")
|
||||
if ceiling < 1:
|
||||
raise ValueError(f"{field} must be at least 1, got {ceiling}")
|
||||
return ceiling
|
||||
|
|
@ -5,6 +5,10 @@ from typing import Final, cast
|
|||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
DEFAULT_MAX_AGENTIC_LOOPS,
|
||||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
CHAT_COMPLETION_AGENTIC_SURFACE,
|
||||
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
|
||||
|
|
@ -52,7 +56,10 @@ def _coerce_int(value: object, default: int) -> int:
|
|||
|
||||
def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]:
|
||||
depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0)
|
||||
max_loops: Final = max(_coerce_int(kwargs.get("max_agentic_loops"), 3), 1)
|
||||
configured: Final = validated_max_agentic_loops(
|
||||
kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
|
||||
)
|
||||
max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
|
||||
raw_fingerprints: Final = kwargs.get("_agentic_loop_fingerprints")
|
||||
fingerprints: Final = [str(fp) for fp in raw_fingerprints] if isinstance(raw_fingerprints, list) else []
|
||||
return depth, max_loops, fingerprints
|
||||
|
|
|
|||
|
|
@ -113,6 +113,14 @@ class FakeAnthropicMessagesStreamIterator:
|
|||
}
|
||||
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
|
||||
|
||||
else:
|
||||
passthrough_start: Final = {
|
||||
"type": "content_block_start",
|
||||
"index": index,
|
||||
"content_block": block_dict,
|
||||
}
|
||||
chunks.append(f"event: content_block_start\ndata: {json.dumps(passthrough_start)}\n\n".encode())
|
||||
|
||||
content_block_stop: Final = {"type": "content_block_stop", "index": index}
|
||||
chunks.append(f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode())
|
||||
return chunks
|
||||
|
|
|
|||
|
|
@ -19,6 +19,10 @@ import litellm.types.utils
|
|||
from litellm._logging import _redact_string, verbose_logger
|
||||
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
|
||||
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
|
||||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
DEFAULT_MAX_AGENTIC_LOOPS,
|
||||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason
|
||||
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
|
||||
|
|
@ -89,6 +93,7 @@ from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadCon
|
|||
from litellm.types.integrations.custom_logger import (
|
||||
AgenticLoopPlan,
|
||||
AgenticLoopRequestPatch,
|
||||
AgenticLoopSafetyError,
|
||||
)
|
||||
from litellm.types.llms.anthropic_messages.anthropic_response import (
|
||||
AnthropicMessagesResponse,
|
||||
|
|
@ -5077,9 +5082,12 @@ class BaseLLMHTTPHandler:
|
|||
@staticmethod
|
||||
def _get_agentic_loop_settings(kwargs: dict) -> tuple[int, int, list[str]]:
|
||||
depth: Final = int(kwargs.get("_agentic_loop_depth", 0) or 0)
|
||||
max_loops: Final = int(kwargs.get("max_agentic_loops", 3) or 3)
|
||||
configured: Final = validated_max_agentic_loops(
|
||||
kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
|
||||
)
|
||||
max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
|
||||
fingerprints: Final = list(kwargs.get("_agentic_loop_fingerprints", []) or [])
|
||||
return depth, max(max_loops, 1), fingerprints
|
||||
return depth, max_loops, fingerprints
|
||||
|
||||
@staticmethod
|
||||
def _has_agentic_completion_hook(logging_obj: LiteLLMLoggingObj) -> bool:
|
||||
|
|
@ -5122,7 +5130,8 @@ class BaseLLMHTTPHandler:
|
|||
"""
|
||||
Evaluate agentic-loop safety guards (fingerprint cycle / max depth).
|
||||
|
||||
Raises ValueError on abort. Returns the current fingerprint on success.
|
||||
Raises AgenticLoopSafetyError on abort. Returns the current fingerprint
|
||||
on success.
|
||||
|
||||
These checks must not be swallowed by the per-callback ``except Exception``
|
||||
block that wraps callback dispatch — they are bounded-loop / cycle-break
|
||||
|
|
@ -5130,9 +5139,9 @@ class BaseLLMHTTPHandler:
|
|||
"""
|
||||
fingerprint: Final = BaseLLMHTTPHandler._fingerprint_agentic_tools(tool_calls)
|
||||
if fingerprint in fingerprints:
|
||||
raise ValueError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
|
||||
raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
|
||||
if depth >= max_loops:
|
||||
raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
|
||||
raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
|
||||
return fingerprint
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -5142,6 +5151,97 @@ class BaseLLMHTTPHandler:
|
|||
except Exception:
|
||||
return str(tools)
|
||||
|
||||
@staticmethod
|
||||
def _refused_agentic_tool_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
|
||||
"""
|
||||
Collect the ids and names of the tool calls a safety rail just refused.
|
||||
|
||||
Callbacks hand back either a bare list of tool calls or a dict wrapping
|
||||
that list under ``tool_calls``, and both the anthropic and responses
|
||||
shapes carry an ``id`` (or ``call_id``) plus a ``name``.
|
||||
"""
|
||||
calls: Final = tool_calls.get("tool_calls") if isinstance(tool_calls, dict) else tool_calls
|
||||
if not isinstance(calls, list):
|
||||
return frozenset(), frozenset()
|
||||
dict_calls: Final = (call for call in calls if isinstance(call, dict))
|
||||
fields: Final = tuple((call.get("id"), call.get("call_id"), call.get("name")) for call in dict_calls)
|
||||
ids: Final = frozenset(
|
||||
value for call_id, caller_id, _ in fields for value in (call_id, caller_id) if isinstance(value, str)
|
||||
)
|
||||
names: Final = frozenset(name for _, _, name in fields if isinstance(name, str))
|
||||
return ids, names
|
||||
|
||||
@staticmethod
|
||||
def _is_refused_tool_use_block(block: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
|
||||
"""
|
||||
Whether this response block belongs to a tool call the rail refused.
|
||||
|
||||
An id settles it on its own, so a block carrying one is matched on the id
|
||||
alone and a client's own tool call survives even where it happens to
|
||||
share a name with a refused one. The name is only consulted for tool call
|
||||
shapes that arrive without an id.
|
||||
"""
|
||||
if not isinstance(block, dict) or block.get("type") != "tool_use":
|
||||
return False
|
||||
block_id: Final = block.get("id")
|
||||
if isinstance(block_id, str) and refused_ids:
|
||||
return block_id in refused_ids
|
||||
return block.get("name") in refused_names
|
||||
|
||||
@staticmethod
|
||||
def _can_replace_turn_with_terminal_response(stream: bool, api_surface: str) -> bool:
|
||||
"""
|
||||
Whether a refused rerun can still be answered with a finalized turn.
|
||||
|
||||
Only the anthropic messages surface can. The responses surface carries a
|
||||
pydantic model the finalizer does not rewrite, so it keeps raising, which
|
||||
is what every surface did before this path learned to end the turn.
|
||||
|
||||
The messages and responses call sites pass ``stream=False``, because
|
||||
interception converts an intercepted stream to non-streaming before the
|
||||
loop runs and rebuilds the SSE stream from the finalized turn
|
||||
afterwards. ``AgenticStreamingIterator`` passes ``stream=True``, and
|
||||
that path keeps raising: its events are already on the wire, so a
|
||||
finalized turn would reach the client as a second message rather than
|
||||
as a replacement.
|
||||
"""
|
||||
return not stream and api_surface == "anthropic_messages"
|
||||
|
||||
@staticmethod
|
||||
def _finalize_refused_agentic_response(response: object, tool_calls: object) -> object:
|
||||
"""
|
||||
Turn the response into a terminal turn after a safety rail refused the rerun.
|
||||
|
||||
The refused tool calls target tools LiteLLM injected on the client's
|
||||
behalf, so a client that never declared them cannot send back a matching
|
||||
``tool_result``. Their blocks are dropped and a ``tool_use`` stop reason
|
||||
is closed out as ``end_turn``, which is what a provider-native web search
|
||||
turn returns once it stops calling tools.
|
||||
|
||||
A ``tool_use`` block the client itself declared is left alone, and while
|
||||
one is still in the response the stop reason stays ``tool_use`` so the
|
||||
client knows to answer it.
|
||||
"""
|
||||
if not isinstance(response, dict):
|
||||
return response
|
||||
|
||||
refused_ids, refused_names = BaseLLMHTTPHandler._refused_agentic_tool_identifiers(tool_calls)
|
||||
finalized: Final = dict(response)
|
||||
content: Final = finalized.get("content")
|
||||
if isinstance(content, list):
|
||||
kept_blocks: Final = [
|
||||
block
|
||||
for block in content
|
||||
if not BaseLLMHTTPHandler._is_refused_tool_use_block(block, refused_ids, refused_names)
|
||||
]
|
||||
finalized["content"] = kept_blocks
|
||||
client_tool_use_remains: Final = any(
|
||||
isinstance(block, dict) and block.get("type") == "tool_use" for block in kept_blocks
|
||||
)
|
||||
if not client_tool_use_remains and finalized.get("stop_reason") == "tool_use":
|
||||
finalized["stop_reason"] = "end_turn"
|
||||
return finalized
|
||||
|
||||
async def _execute_anthropic_agentic_plan(
|
||||
self,
|
||||
plan: AgenticLoopPlan,
|
||||
|
|
@ -5507,14 +5607,30 @@ class BaseLLMHTTPHandler:
|
|||
continue
|
||||
|
||||
# Safety guards must run OUTSIDE the callback try/except — they are
|
||||
# bounded-loop / cycle-break rails that must propagate to the caller.
|
||||
fingerprint = self._check_agentic_loop_safety(
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model=model,
|
||||
)
|
||||
# bounded-loop / cycle-break rails, not callback bugs.
|
||||
try:
|
||||
fingerprint = self._check_agentic_loop_safety(
|
||||
tool_calls=tool_calls,
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model=model,
|
||||
)
|
||||
except AgenticLoopSafetyError as e:
|
||||
if not self._can_replace_turn_with_terminal_response(stream, api_surface):
|
||||
raise
|
||||
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
|
||||
verbose_logger.warning(
|
||||
"LiteLLM.AgenticLoopRefused: ending turn [call_id=%s model=%s]: %s",
|
||||
_call_id,
|
||||
model,
|
||||
str(e),
|
||||
)
|
||||
return self._maybe_wrap_in_fake_stream(
|
||||
self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls),
|
||||
logging_obj,
|
||||
api_surface,
|
||||
)
|
||||
|
||||
try:
|
||||
kwargs_with_provider = hook_kwargs.copy()
|
||||
|
|
|
|||
|
|
@ -254,6 +254,9 @@ from litellm.exceptions import RejectedRequestError
|
|||
from litellm.integrations.custom_guardrail import ModifyResponseException
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
|
||||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
_get_parent_otel_span_from_kwargs,
|
||||
|
|
@ -4081,6 +4084,27 @@ def resolve_complexity_router_plugins(
|
|||
complexity_router_config["classifier_plugin"] = resolved_classifier # rebind-ok: out-param, resolved in place
|
||||
|
||||
|
||||
def validate_deployment_max_agentic_loops(model: Mapping[str, Any]) -> None:
|
||||
"""
|
||||
Reject a per-deployment `max_agentic_loops` the agentic loop cannot honor.
|
||||
|
||||
Checked here rather than on `LiteLLM_Params` because the proxy builds its
|
||||
router with `ignore_invalid_deployments=True`, so a validator down there
|
||||
turns a bad value into a silently missing model instead of a refusal to
|
||||
start. Left unchecked entirely, a `0` used to read as the default ceiling
|
||||
of 3 and a non-integer failed every request to that model instead.
|
||||
"""
|
||||
litellm_params: Final = model.get("litellm_params") or {}
|
||||
if "max_agentic_loops" not in litellm_params:
|
||||
return
|
||||
|
||||
model_name: Final = model.get("model_name", "")
|
||||
validated_max_agentic_loops(
|
||||
litellm_params["max_agentic_loops"],
|
||||
field=f"litellm_params.max_agentic_loops on model {model_name!r}",
|
||||
)
|
||||
|
||||
|
||||
def pin_complexity_router_model_id(model: dict) -> None: # mutable-ok: out-param, model_info is stamped in place
|
||||
"""
|
||||
Stamps `model_info.id` from the raw litellm_params before plugin resolution swaps
|
||||
|
|
@ -5416,6 +5440,7 @@ class ProxyConfig:
|
|||
for k, v in model["litellm_params"].items():
|
||||
if isinstance(v, str) and v.startswith("os.environ/"):
|
||||
model["litellm_params"][k] = get_secret(v)
|
||||
validate_deployment_max_agentic_loops(model)
|
||||
pin_complexity_router_model_id(model)
|
||||
complexity_router_config = model["litellm_params"].get("complexity_router_config")
|
||||
if isinstance(complexity_router_config, dict):
|
||||
|
|
|
|||
|
|
@ -23,6 +23,21 @@ def is_interception_internal_key(
|
|||
return any(key.startswith(prefix) for prefix in prefixes)
|
||||
|
||||
|
||||
class AgenticLoopSafetyError(ValueError):
|
||||
"""
|
||||
Raised when an agentic-loop safety rail refuses a rerun.
|
||||
|
||||
Covers both rails: the bounded-loop cap (``max_agentic_loops``) and the
|
||||
repeated tool-call fingerprint cycle break. Subclasses ``ValueError`` so
|
||||
callers that already catch the broader type keep working.
|
||||
|
||||
Only the anthropic messages loop raises this today. The chat completions
|
||||
loop in ``litellm_core_utils/chat_completion_agentic_loop.py`` still raises
|
||||
a plain ``ValueError`` from its own copy of the same rails, so catching
|
||||
this type alone will not cover that surface until it is moved over.
|
||||
"""
|
||||
|
||||
|
||||
class StandardCustomLoggerInitParams(BaseModel):
|
||||
"""
|
||||
Params for initializing a CustomLogger.
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ Type definitions for WebSearch Interception integration.
|
|||
from typing import Literal, TypedDict
|
||||
|
||||
from pydantic import BaseModel
|
||||
from typing_extensions import ReadOnly
|
||||
|
||||
|
||||
class AnthropicSearchQuery(BaseModel):
|
||||
|
|
@ -35,6 +36,7 @@ class WebSearchInterceptionConfig(TypedDict, total=False):
|
|||
websearch_interception_params:
|
||||
enabled_providers: ["bedrock"]
|
||||
search_tool_name: "my-perplexity-search"
|
||||
max_agentic_loops: 5
|
||||
"""
|
||||
|
||||
enabled_providers: list[str]
|
||||
|
|
@ -42,3 +44,6 @@ class WebSearchInterceptionConfig(TypedDict, total=False):
|
|||
|
||||
search_tool_name: str | None
|
||||
"""Name of search tool configured in router's search_tools. If None, uses first available."""
|
||||
|
||||
max_agentic_loops: ReadOnly[int | None]
|
||||
"""How many follow-up model calls one intercepted request may chain. If None, LiteLLM's default of 3 applies."""
|
||||
|
|
|
|||
|
|
@ -0,0 +1,754 @@
|
|||
"""
|
||||
Unit tests for what an intercepted request returns once a safety rail refuses
|
||||
another agentic loop.
|
||||
|
||||
The web search interception loop injects an internal tool (litellm_web_search)
|
||||
that the client never declared. When the loop cap or the repeated-fingerprint
|
||||
guard trips, the turn has to end with a terminal response: leaking that internal
|
||||
tool_use block leaves the client holding a tool call it cannot answer.
|
||||
|
||||
Also covers the max_agentic_loops knob on websearch_interception_params, from
|
||||
config.yaml through to the settings the loop actually reads.
|
||||
"""
|
||||
|
||||
import json
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.websearch_interception.handler import (
|
||||
WebSearchInterceptionLogger,
|
||||
)
|
||||
from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
|
||||
FakeAnthropicMessagesStreamIterator,
|
||||
)
|
||||
from litellm.litellm_core_utils.agentic_loop_settings import DEFAULT_MAX_AGENTIC_LOOPS
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.secret_managers.main import get_secret
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
AgenticLoopPlan,
|
||||
AgenticLoopRequestPatch,
|
||||
AgenticLoopSafetyError,
|
||||
)
|
||||
|
||||
INTERNAL_TOOL_NAME = "litellm_web_search"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def only_the_callbacks_these_tests_register(monkeypatch):
|
||||
"""
|
||||
These tests drive the hooks with a callback of their own on the logging
|
||||
object, so a logger another test left on litellm.callbacks would join the
|
||||
run and change what the hooks do.
|
||||
"""
|
||||
monkeypatch.setattr(litellm, "callbacks", [])
|
||||
|
||||
|
||||
def _internal_tool_use_block(block_id: str = "toolu_internal_1") -> dict:
|
||||
return {
|
||||
"id": block_id,
|
||||
"type": "tool_use",
|
||||
"name": INTERNAL_TOOL_NAME,
|
||||
"input": {"query": "who won the world cup"},
|
||||
}
|
||||
|
||||
|
||||
def _native_search_blocks(index: int = 1) -> list[dict]:
|
||||
return [
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": f"srvtoolu_{index}",
|
||||
"name": "web_search",
|
||||
"input": {"query": "who won the world cup"},
|
||||
},
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": f"srvtoolu_{index}",
|
||||
"content": [{"type": "web_search_result", "url": "https://example.com", "title": "Result"}],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _response_asking_for_another_search(block_id: str = "toolu_internal_1") -> dict:
|
||||
return {
|
||||
"id": "msg_123",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-5",
|
||||
"content": [
|
||||
*_native_search_blocks(index=1),
|
||||
{"type": "text", "text": "Let me check one more source."},
|
||||
_internal_tool_use_block(block_id),
|
||||
],
|
||||
"stop_reason": "tool_use",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
}
|
||||
|
||||
|
||||
def _block_types(response: dict) -> list[str]:
|
||||
return [block["type"] for block in response["content"]]
|
||||
|
||||
|
||||
def _tool_use_names(response: dict) -> list[str]:
|
||||
return [block.get("name") for block in response["content"] if block.get("type") == "tool_use"]
|
||||
|
||||
|
||||
class _InterceptingCallback(CustomLogger):
|
||||
"""
|
||||
Stands in for the websearch interceptor: asks for another loop whenever the
|
||||
response carries an internal web search tool_use block, and injects the
|
||||
native block pair on the way back out.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.plan_calls = 0
|
||||
self.post_hook_calls = 0
|
||||
|
||||
async def async_should_run_agentic_loop(
|
||||
self, response, model, messages, tools, stream, custom_llm_provider, kwargs
|
||||
):
|
||||
if not isinstance(response, dict):
|
||||
return True, {"tool_calls": [_internal_tool_use_block()]}
|
||||
tool_calls = [
|
||||
block
|
||||
for block in response.get("content", [])
|
||||
if block.get("type") == "tool_use" and block.get("name") == INTERNAL_TOOL_NAME
|
||||
]
|
||||
if not tool_calls:
|
||||
return False, {}
|
||||
return True, {"tool_calls": tool_calls, "tool_type": "websearch"}
|
||||
|
||||
async def async_build_agentic_loop_plan(
|
||||
self,
|
||||
tools,
|
||||
model,
|
||||
messages,
|
||||
response,
|
||||
anthropic_messages_provider_config,
|
||||
anthropic_messages_optional_request_params,
|
||||
logging_obj,
|
||||
stream,
|
||||
kwargs,
|
||||
):
|
||||
self.plan_calls += 1
|
||||
return AgenticLoopPlan(
|
||||
run_agentic_loop=True,
|
||||
request_patch=AgenticLoopRequestPatch(
|
||||
messages=[{"role": "user", "content": "here are the search results"}],
|
||||
max_tokens=1024,
|
||||
),
|
||||
)
|
||||
|
||||
async def async_post_agentic_loop_response_hook(self, response, plan, kwargs):
|
||||
self.post_hook_calls += 1
|
||||
if isinstance(response, dict):
|
||||
response["content"] = [*_native_search_blocks(index=2), *response.get("content", [])]
|
||||
return response
|
||||
|
||||
|
||||
def _logging_obj(callback: CustomLogger, converted_stream: bool = False) -> MagicMock:
|
||||
logging_obj = MagicMock()
|
||||
logging_obj.model_call_details = {"websearch_interception_converted_stream": converted_stream}
|
||||
logging_obj.dynamic_success_callbacks = [callback]
|
||||
logging_obj.litellm_call_id = "call-abc"
|
||||
return logging_obj
|
||||
|
||||
|
||||
async def _run_hooks(
|
||||
handler: BaseLLMHTTPHandler,
|
||||
callback: CustomLogger,
|
||||
kwargs: dict,
|
||||
response: object = None,
|
||||
stream: bool = False,
|
||||
converted_stream: bool = False,
|
||||
api_surface: str = "anthropic_messages",
|
||||
):
|
||||
return await handler._call_agentic_completion_hooks(
|
||||
response=_response_asking_for_another_search() if response is None else response,
|
||||
model="claude-sonnet-4-5",
|
||||
messages=[{"role": "user", "content": "who won the world cup"}],
|
||||
anthropic_messages_provider_config=MagicMock(),
|
||||
anthropic_messages_optional_request_params={},
|
||||
logging_obj=_logging_obj(callback, converted_stream=converted_stream),
|
||||
stream=stream,
|
||||
custom_llm_provider="anthropic",
|
||||
kwargs=kwargs,
|
||||
api_surface=api_surface,
|
||||
)
|
||||
|
||||
|
||||
class TestCappedLoopReturnsTerminalResponse:
|
||||
def setup_method(self):
|
||||
self.handler = BaseLLMHTTPHandler()
|
||||
self.callback = _InterceptingCallback()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_internal_tool_use_block_is_dropped(self):
|
||||
result = await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
)
|
||||
|
||||
assert isinstance(result, dict)
|
||||
assert INTERNAL_TOOL_NAME not in _tool_use_names(result)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stop_reason_is_closed_out(self):
|
||||
result = await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
)
|
||||
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_blocks_and_text_survive(self):
|
||||
result = await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
)
|
||||
|
||||
assert _block_types(result) == ["server_tool_use", "web_search_tool_result", "text"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_turn_carrying_only_the_refused_call_still_ends_cleanly(self):
|
||||
"""
|
||||
The refused call can be every block the model produced, which leaves the
|
||||
turn with no content once it is dropped. That still has to come back as a
|
||||
finished turn rather than as the leaked call, so the client stops instead
|
||||
of waiting on a tool it cannot run, and the rest of the message survives
|
||||
so the request is still billed and traceable.
|
||||
|
||||
An empty turn renders as nothing, which is the ceiling being set too low
|
||||
for the question rather than a malformed response.
|
||||
"""
|
||||
nothing_but_the_refused_call = {
|
||||
"id": "msg_123",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-5",
|
||||
"content": [_internal_tool_use_block()],
|
||||
"stop_reason": "tool_use",
|
||||
"usage": {"input_tokens": 10, "output_tokens": 5},
|
||||
}
|
||||
|
||||
result = await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
response=nothing_but_the_refused_call,
|
||||
)
|
||||
|
||||
assert result["content"] == []
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
assert result["usage"] == {"input_tokens": 10, "output_tokens": 5}
|
||||
assert result["id"] == "msg_123"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_follow_up_model_call_is_planned(self):
|
||||
"""
|
||||
The rail has to end the turn without planning another model call, and it
|
||||
has to end it by returning rather than by raising, which is the half that
|
||||
the caller's response depends on.
|
||||
"""
|
||||
result = await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
)
|
||||
|
||||
assert self.callback.plan_calls == 0
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_original_response_is_not_mutated(self):
|
||||
response = _response_asking_for_another_search()
|
||||
|
||||
await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
response=response,
|
||||
)
|
||||
|
||||
assert response["stop_reason"] == "tool_use"
|
||||
assert INTERNAL_TOOL_NAME in _tool_use_names(response)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_repeated_fingerprint_guard_is_terminal_too(self):
|
||||
tool_calls = {"tool_calls": [_internal_tool_use_block()], "tool_type": "websearch"}
|
||||
seen = json.dumps(tool_calls, sort_keys=True, default=str)
|
||||
|
||||
result = await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, "_agentic_loop_fingerprints": [seen]},
|
||||
)
|
||||
|
||||
assert self.callback.plan_calls == 0
|
||||
assert INTERNAL_TOOL_NAME not in _tool_use_names(result)
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_client_declared_tool_use_is_left_alone(self):
|
||||
response = _response_asking_for_another_search()
|
||||
client_tool_use = {"id": "toolu_client_1", "type": "tool_use", "name": "get_weather", "input": {}}
|
||||
response["content"].append(client_tool_use)
|
||||
|
||||
result = await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
response=response,
|
||||
)
|
||||
|
||||
assert _tool_use_names(result) == ["get_weather"]
|
||||
assert result["stop_reason"] == "tool_use"
|
||||
|
||||
def test_only_the_refused_tool_calls_are_dropped(self):
|
||||
"""
|
||||
A block is matched on the id the rail refused, not on the tool name, so a
|
||||
second block sharing that name survives when the rail never listed it. A
|
||||
callback that picks its tool calls out by name hands both over and both
|
||||
go, which is its own call to make; this is about not widening it here.
|
||||
"""
|
||||
response = _response_asking_for_another_search()
|
||||
response["content"].append(
|
||||
{"id": "toolu_client_1", "type": "tool_use", "name": INTERNAL_TOOL_NAME, "input": {}}
|
||||
)
|
||||
|
||||
result = BaseLLMHTTPHandler._finalize_refused_agentic_response(
|
||||
response=response,
|
||||
tool_calls={"tool_calls": [_internal_tool_use_block()]},
|
||||
)
|
||||
|
||||
assert [block["id"] for block in result["content"] if block.get("type") == "tool_use"] == ["toolu_client_1"]
|
||||
assert result["stop_reason"] == "tool_use"
|
||||
|
||||
def test_tool_calls_without_ids_still_match_by_name(self):
|
||||
"""
|
||||
Not every callback shape carries ids on its tool calls, so the name is
|
||||
still what decides when the rail refused a call that has no id.
|
||||
"""
|
||||
result = BaseLLMHTTPHandler._finalize_refused_agentic_response(
|
||||
response=_response_asking_for_another_search(),
|
||||
tool_calls={"tool_calls": [{"name": INTERNAL_TOOL_NAME, "input": {}}]},
|
||||
)
|
||||
|
||||
assert _tool_use_names(result) == []
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_streaming_caller_is_left_to_its_existing_behavior(self):
|
||||
"""
|
||||
A streaming caller has already sent the original message to the client, so
|
||||
a finalized turn would land as a second message rather than replace the
|
||||
first. The rail keeps raising there and the caller handles it as before.
|
||||
"""
|
||||
with pytest.raises(AgenticLoopSafetyError):
|
||||
await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
stream=True,
|
||||
)
|
||||
|
||||
assert self.callback.plan_calls == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_surface_is_left_to_its_existing_behavior(self):
|
||||
"""
|
||||
The responses surface carries a pydantic model rather than the anthropic
|
||||
dict this finalizer rewrites, so it keeps raising instead of being handed
|
||||
a response that was never actually finalized.
|
||||
"""
|
||||
with pytest.raises(AgenticLoopSafetyError):
|
||||
await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
api_surface="responses",
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_dict_response_is_returned_untouched(self):
|
||||
response = MagicMock()
|
||||
|
||||
result = await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
response=response,
|
||||
)
|
||||
|
||||
assert result is response
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_converted_stream_gets_a_terminal_fake_stream(self):
|
||||
"""
|
||||
A converted stream is wrapped back into an Anthropic SSE stream here, the
|
||||
same as every other return in this function, so a streaming client gets a
|
||||
terminal stream rather than a bare dict. The interceptor turns the client's
|
||||
stream into a non-streaming upstream call, so stream is False on this path
|
||||
and the converted flag on the logging object is what marks it.
|
||||
"""
|
||||
result = await _run_hooks(
|
||||
self.handler,
|
||||
self.callback,
|
||||
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
|
||||
converted_stream=True,
|
||||
)
|
||||
|
||||
assert isinstance(result, FakeAnthropicMessagesStreamIterator)
|
||||
assert result.response["stop_reason"] == "end_turn"
|
||||
assert INTERNAL_TOOL_NAME not in _tool_use_names(result.response)
|
||||
|
||||
def test_rails_cannot_trip_in_the_outermost_frame(self):
|
||||
"""
|
||||
Backs the invariant the test above relies on: at depth 0 the fingerprint set
|
||||
is empty and the ceiling is at least 1, so neither rail can refuse.
|
||||
"""
|
||||
depth, max_loops, fingerprints = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={})
|
||||
|
||||
assert depth == 0
|
||||
assert fingerprints == []
|
||||
assert max_loops >= 1
|
||||
|
||||
depth, max_loops, fingerprints = BaseLLMHTTPHandler._get_agentic_loop_settings(
|
||||
kwargs={"max_agentic_loops": 1}
|
||||
)
|
||||
|
||||
assert max_loops == 1
|
||||
assert BaseLLMHTTPHandler._check_agentic_loop_safety(
|
||||
tool_calls={"tool_calls": [_internal_tool_use_block()]},
|
||||
fingerprints=fingerprints,
|
||||
depth=depth,
|
||||
max_loops=max_loops,
|
||||
model="claude-sonnet-4-5",
|
||||
)
|
||||
|
||||
def test_safety_error_is_still_a_value_error(self):
|
||||
assert issubclass(AgenticLoopSafetyError, ValueError)
|
||||
|
||||
def test_safety_error_type_names_the_rail(self):
|
||||
with pytest.raises(AgenticLoopSafetyError, match="max_agentic_loops"):
|
||||
BaseLLMHTTPHandler._check_agentic_loop_safety(
|
||||
tool_calls={"tool_calls": [_internal_tool_use_block()]},
|
||||
fingerprints=[],
|
||||
depth=3,
|
||||
max_loops=3,
|
||||
model="claude-sonnet-4-5",
|
||||
)
|
||||
|
||||
|
||||
class TestOuterFramePostHookStillRuns:
|
||||
"""
|
||||
The cap used to raise through the parent frame's await, which skipped the
|
||||
parent's post-loop hook. The parent now gets its terminal response back and
|
||||
finishes normally, so the blocks it was going to inject still land.
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_parent_frame_injects_its_blocks_after_the_cap_trips(self, monkeypatch):
|
||||
handler = BaseLLMHTTPHandler()
|
||||
callback = _InterceptingCallback()
|
||||
|
||||
async def fake_acreate(**call_kwargs):
|
||||
return await handler._call_agentic_completion_hooks(
|
||||
response=_response_asking_for_another_search(block_id="toolu_internal_2"),
|
||||
model=call_kwargs["model"],
|
||||
messages=call_kwargs["messages"],
|
||||
anthropic_messages_provider_config=MagicMock(),
|
||||
anthropic_messages_optional_request_params={},
|
||||
logging_obj=_logging_obj(callback),
|
||||
stream=False,
|
||||
custom_llm_provider="anthropic",
|
||||
kwargs={
|
||||
key: call_kwargs[key]
|
||||
for key in ("_agentic_loop_depth", "max_agentic_loops", "_agentic_loop_fingerprints")
|
||||
if key in call_kwargs
|
||||
},
|
||||
)
|
||||
|
||||
monkeypatch.setattr("litellm.anthropic_interface.messages.acreate", fake_acreate)
|
||||
|
||||
result = await _run_hooks(
|
||||
handler,
|
||||
callback,
|
||||
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 1},
|
||||
)
|
||||
|
||||
assert callback.plan_calls == 1
|
||||
assert callback.post_hook_calls == 1
|
||||
assert _block_types(result)[:2] == ["server_tool_use", "web_search_tool_result"]
|
||||
assert INTERNAL_TOOL_NAME not in _tool_use_names(result)
|
||||
assert result["stop_reason"] == "end_turn"
|
||||
|
||||
|
||||
class TestMaxAgenticLoopsConfigKnob:
|
||||
def test_from_config_yaml_reads_the_knob(self):
|
||||
logger = WebSearchInterceptionLogger.from_config_yaml(
|
||||
{"enabled_providers": ["bedrock"], "max_agentic_loops": 7}
|
||||
)
|
||||
|
||||
assert logger.max_agentic_loops == 7
|
||||
|
||||
def test_from_config_yaml_leaves_it_unset_by_default(self):
|
||||
logger = WebSearchInterceptionLogger.from_config_yaml({"enabled_providers": ["bedrock"]})
|
||||
|
||||
assert logger.max_agentic_loops is None
|
||||
|
||||
@pytest.mark.parametrize("bad_value", [0, -1])
|
||||
def test_out_of_range_ceilings_are_rejected_at_config_load(self, bad_value):
|
||||
with pytest.raises(ValueError, match="max_agentic_loops"):
|
||||
WebSearchInterceptionLogger.from_config_yaml(
|
||||
{"enabled_providers": ["bedrock"], "max_agentic_loops": bad_value}
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("bad_value", ["three", True, 2.5])
|
||||
def test_non_integer_ceilings_are_rejected_at_config_load(self, bad_value):
|
||||
with pytest.raises(TypeError, match="max_agentic_loops"):
|
||||
WebSearchInterceptionLogger.from_config_yaml(
|
||||
{"enabled_providers": ["bedrock"], "max_agentic_loops": bad_value}
|
||||
)
|
||||
|
||||
def test_a_ceiling_spelled_as_a_string_is_read_at_config_load(self):
|
||||
"""
|
||||
`max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS` resolves to a string
|
||||
before it reaches the knob, so refusing "5" would break a config that
|
||||
works today.
|
||||
"""
|
||||
logger = WebSearchInterceptionLogger.from_config_yaml(
|
||||
{"enabled_providers": ["bedrock"], "max_agentic_loops": "5"}
|
||||
)
|
||||
|
||||
assert logger.max_agentic_loops == 5
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_knob_reaches_the_loop_settings(self):
|
||||
logger = WebSearchInterceptionLogger.from_config_yaml(
|
||||
{"enabled_providers": ["bedrock"], "max_agentic_loops": 7}
|
||||
)
|
||||
kwargs = {
|
||||
"tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}],
|
||||
"litellm_params": {"custom_llm_provider": "bedrock"},
|
||||
}
|
||||
|
||||
updated = await logger.async_pre_request_hook(model="claude-sonnet-4-5", messages=[], kwargs=kwargs)
|
||||
|
||||
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs=updated)
|
||||
assert max_loops == 7
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deployment_setting_wins_over_the_feature_setting(self):
|
||||
logger = WebSearchInterceptionLogger.from_config_yaml(
|
||||
{"enabled_providers": ["bedrock"], "max_agentic_loops": 7}
|
||||
)
|
||||
kwargs = {
|
||||
"tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}],
|
||||
"litellm_params": {"custom_llm_provider": "bedrock"},
|
||||
"max_agentic_loops": 2,
|
||||
}
|
||||
|
||||
updated = await logger.async_pre_request_hook(model="claude-sonnet-4-5", messages=[], kwargs=kwargs)
|
||||
|
||||
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs=updated)
|
||||
assert max_loops == 2
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_default_ceiling_applies_when_the_knob_is_unset(self):
|
||||
logger = WebSearchInterceptionLogger.from_config_yaml({"enabled_providers": ["bedrock"]})
|
||||
kwargs = {
|
||||
"tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}],
|
||||
"litellm_params": {"custom_llm_provider": "bedrock"},
|
||||
}
|
||||
|
||||
updated = await logger.async_pre_request_hook(model="claude-sonnet-4-5", messages=[], kwargs=kwargs)
|
||||
|
||||
assert "max_agentic_loops" not in updated
|
||||
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs=updated)
|
||||
assert max_loops == 3
|
||||
|
||||
|
||||
def _stream_events(response: dict) -> list[dict]:
|
||||
events: list[dict] = []
|
||||
for chunk in FakeAnthropicMessagesStreamIterator(response=response):
|
||||
for line in chunk.decode().splitlines():
|
||||
if line.startswith("data: "):
|
||||
events.append(json.loads(line[len("data: ") :]))
|
||||
return events
|
||||
|
||||
|
||||
class TestBothCeilingKnobsAreValidated:
|
||||
"""
|
||||
``max_agentic_loops`` is settable per deployment and feature-wide, and the
|
||||
per-deployment one wins. Only the feature-wide one used to be checked, so a
|
||||
per-deployment ``0`` was swallowed by an ``or 3`` and read as the default 3,
|
||||
handing the loosest ceiling to whoever asked for the tightest.
|
||||
"""
|
||||
|
||||
def test_a_per_deployment_zero_is_rejected_not_read_as_the_default(self):
|
||||
with pytest.raises(ValueError, match="must be at least 1, got 0"):
|
||||
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 0})
|
||||
|
||||
def test_a_per_deployment_non_integer_names_the_field_it_came_from(self):
|
||||
with pytest.raises(TypeError, match=r"litellm_params\.max_agentic_loops must be an integer"):
|
||||
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": "three"})
|
||||
|
||||
def test_a_per_deployment_true_is_not_read_as_a_ceiling_of_one(self):
|
||||
with pytest.raises(TypeError, match="must be an integer"):
|
||||
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": True})
|
||||
|
||||
def test_an_absent_ceiling_falls_back_to_the_shared_default(self):
|
||||
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={})
|
||||
|
||||
assert max_loops == DEFAULT_MAX_AGENTIC_LOOPS
|
||||
|
||||
def test_an_explicit_none_falls_back_to_the_shared_default(self):
|
||||
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": None})
|
||||
|
||||
assert max_loops == DEFAULT_MAX_AGENTIC_LOOPS
|
||||
|
||||
def test_a_valid_per_deployment_ceiling_is_passed_through(self):
|
||||
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 6})
|
||||
|
||||
assert max_loops == 6
|
||||
|
||||
@pytest.mark.parametrize("rejected", [0, -1, "three", True])
|
||||
def test_the_two_knobs_reject_the_same_values(self, rejected):
|
||||
with pytest.raises((TypeError, ValueError)):
|
||||
WebSearchInterceptionLogger(max_agentic_loops=rejected)
|
||||
with pytest.raises((TypeError, ValueError)):
|
||||
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": rejected})
|
||||
|
||||
def test_each_knob_names_its_own_config_field(self):
|
||||
with pytest.raises(ValueError, match=r"websearch_interception_params\.max_agentic_loops"):
|
||||
WebSearchInterceptionLogger(max_agentic_loops=0)
|
||||
with pytest.raises(ValueError, match=r"litellm_params\.max_agentic_loops"):
|
||||
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 0})
|
||||
|
||||
|
||||
class TestACeilingThatSpellsAWholeNumberStillWorks:
|
||||
"""
|
||||
The ceiling used to go through ``int(... or 3)``, which accepted anything
|
||||
``int()`` accepted. A ceiling is routinely parameterized as
|
||||
``max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS``, and ``get_secret``
|
||||
hands that back as the string ``"5"``, so tightening the check to
|
||||
``isinstance(int)`` would stop such a proxy from booting on upgrade.
|
||||
"""
|
||||
|
||||
@pytest.mark.parametrize("spelled", ["5", " 5 ", 5.0])
|
||||
def test_a_ceiling_that_spells_five_is_accepted_by_both_knobs(self, spelled):
|
||||
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": spelled})
|
||||
|
||||
assert max_loops == 5
|
||||
assert WebSearchInterceptionLogger(max_agentic_loops=spelled).max_agentic_loops == 5
|
||||
|
||||
def test_an_env_var_sourced_ceiling_survives_secret_resolution(self, monkeypatch):
|
||||
monkeypatch.setenv("MAX_AGENTIC_LOOPS_UNDER_TEST", "7")
|
||||
resolved = get_secret("os.environ/MAX_AGENTIC_LOOPS_UNDER_TEST")
|
||||
|
||||
assert isinstance(resolved, str)
|
||||
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": resolved})
|
||||
assert max_loops == 7
|
||||
|
||||
def test_a_spelled_zero_is_still_refused_and_reports_the_number(self):
|
||||
with pytest.raises(ValueError, match="must be at least 1, got 0"):
|
||||
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": "0"})
|
||||
|
||||
def test_a_word_is_still_refused(self):
|
||||
with pytest.raises(TypeError, match=r"litellm_params\.max_agentic_loops must be an integer"):
|
||||
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": "three"})
|
||||
|
||||
def test_a_fractional_ceiling_is_refused_rather_than_truncated(self):
|
||||
with pytest.raises(TypeError, match="must be an integer"):
|
||||
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 5.5})
|
||||
|
||||
|
||||
class TestRebuiltStreamIsWellFormed:
|
||||
"""
|
||||
A capped turn is rebuilt into SSE by FakeAnthropicMessagesStreamIterator.
|
||||
|
||||
Anthropic's SDK accumulator appends on content_block_start and then indexes
|
||||
content[event.index] on content_block_delta, so a block that stops without
|
||||
ever starting shifts every later index and the accumulator raises
|
||||
IndexError. A web search turn carries server_tool_use and
|
||||
web_search_tool_result blocks, which is exactly where that used to happen.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def _capped_search_turn() -> dict:
|
||||
return {
|
||||
"id": "msg_01",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-5",
|
||||
"stop_reason": "end_turn",
|
||||
"content": [
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_01",
|
||||
"name": "web_search",
|
||||
"input": {"query": "on-demand H100 hourly price"},
|
||||
},
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_01",
|
||||
"content": [
|
||||
{
|
||||
"type": "web_search_result",
|
||||
"url": "https://example.com/h100",
|
||||
"title": "H100 pricing",
|
||||
}
|
||||
],
|
||||
},
|
||||
{"type": "text", "text": "AWS lists the H100 at $12.29 an hour."},
|
||||
],
|
||||
"usage": {"input_tokens": 100, "output_tokens": 20},
|
||||
}
|
||||
|
||||
def test_every_content_block_stop_has_a_matching_start(self):
|
||||
events = _stream_events(self._capped_search_turn())
|
||||
|
||||
started = [event["index"] for event in events if event["type"] == "content_block_start"]
|
||||
stopped = [event["index"] for event in events if event["type"] == "content_block_stop"]
|
||||
|
||||
assert started == [0, 1, 2]
|
||||
assert stopped == [0, 1, 2]
|
||||
|
||||
def test_no_delta_indexes_past_the_blocks_started_before_it(self):
|
||||
events = _stream_events(self._capped_search_turn())
|
||||
|
||||
blocks_started = 0
|
||||
for event in events:
|
||||
if event["type"] == "content_block_start":
|
||||
blocks_started += 1
|
||||
elif event["type"] == "content_block_delta":
|
||||
assert event["index"] < blocks_started
|
||||
|
||||
def test_search_blocks_reach_the_client(self):
|
||||
events = _stream_events(self._capped_search_turn())
|
||||
|
||||
started_types = [
|
||||
event["content_block"]["type"] for event in events if event["type"] == "content_block_start"
|
||||
]
|
||||
|
||||
assert started_types == ["server_tool_use", "web_search_tool_result", "text"]
|
||||
|
||||
def test_the_search_result_survives_the_rebuild_intact(self):
|
||||
events = _stream_events(self._capped_search_turn())
|
||||
|
||||
result_block = next(
|
||||
event["content_block"]
|
||||
for event in events
|
||||
if event["type"] == "content_block_start"
|
||||
and event["content_block"]["type"] == "web_search_tool_result"
|
||||
)
|
||||
|
||||
assert result_block["tool_use_id"] == "srvtoolu_01"
|
||||
assert result_block["content"][0]["url"] == "https://example.com/h100"
|
||||
|
|
@ -26,6 +26,7 @@ from litellm.proxy.proxy_server import (
|
|||
_scrub_guardrail_inner,
|
||||
resolve_complexity_router_plugins,
|
||||
resolve_routing_plugins,
|
||||
validate_deployment_max_agentic_loops,
|
||||
)
|
||||
|
||||
from .conftest import normalize
|
||||
|
|
@ -153,6 +154,71 @@ def test_resolve_complexity_router_plugins_resolves_dotted_path_to_live_instance
|
|||
assert type(config["plugins"][0]).__name__ == "_Plugin"
|
||||
|
||||
|
||||
def test_validate_deployment_max_agentic_loops_allows_a_deployment_without_the_key():
|
||||
model = {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}
|
||||
|
||||
validate_deployment_max_agentic_loops(model)
|
||||
|
||||
assert "max_agentic_loops" not in model["litellm_params"]
|
||||
|
||||
|
||||
def test_validate_deployment_max_agentic_loops_leaves_a_valid_ceiling_alone():
|
||||
model = {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": 5}}
|
||||
|
||||
validate_deployment_max_agentic_loops(model)
|
||||
|
||||
assert model["litellm_params"]["max_agentic_loops"] == 5
|
||||
|
||||
|
||||
def test_validate_deployment_max_agentic_loops_rejects_zero():
|
||||
"""
|
||||
A per-deployment 0 used to be swallowed by an `or 3` and read as the default
|
||||
ceiling of 3, handing the loosest setting to whoever asked for the tightest.
|
||||
"""
|
||||
with pytest.raises(ValueError, match="must be at least 1, got 0"):
|
||||
validate_deployment_max_agentic_loops(
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": 0}}
|
||||
)
|
||||
|
||||
|
||||
def test_validate_deployment_max_agentic_loops_rejects_a_non_integer():
|
||||
"""
|
||||
A per-deployment non-integer used to let the proxy boot and then fail every
|
||||
request to that model with `invalid literal for int() with base 10`.
|
||||
"""
|
||||
with pytest.raises(TypeError, match="must be an integer"):
|
||||
validate_deployment_max_agentic_loops(
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": "three"}}
|
||||
)
|
||||
|
||||
|
||||
def test_validate_deployment_max_agentic_loops_rejects_a_bool():
|
||||
with pytest.raises(TypeError, match="must be an integer"):
|
||||
validate_deployment_max_agentic_loops(
|
||||
{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": True}}
|
||||
)
|
||||
|
||||
|
||||
def test_validate_deployment_max_agentic_loops_accepts_a_ceiling_from_an_env_var():
|
||||
"""
|
||||
`max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS` is resolved to a string
|
||||
before this check runs, and the old `int(... or 3)` accepted that, so
|
||||
refusing it here would stop an already working proxy from booting.
|
||||
"""
|
||||
model = {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": "5"}}
|
||||
|
||||
validate_deployment_max_agentic_loops(model)
|
||||
|
||||
assert model["litellm_params"]["max_agentic_loops"] == "5"
|
||||
|
||||
|
||||
def test_validate_deployment_max_agentic_loops_names_the_offending_model():
|
||||
with pytest.raises(ValueError, match="on model 'claude-sonnet-4-5'"):
|
||||
validate_deployment_max_agentic_loops(
|
||||
{"model_name": "claude-sonnet-4-5", "litellm_params": {"max_agentic_loops": -1}}
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_complexity_router_plugins_rejects_non_routing_plugin_object(tmp_path):
|
||||
plugin_file = tmp_path / "bad_plugin.py"
|
||||
plugin_file.write_text("not_a_plugin = object()\n")
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue