Merge pull request #37911 from BerriAI/litellm_fix_agentic_loop_cap_response

fix(websearch_interception): end the turn when the agentic loop hits its ceiling
This commit is contained in:
Mateo Wang 2026-08-22 11:40:59 -07:00 committed by GitHub
commit abf99e37d6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 1144 additions and 14 deletions

View file

@ -207,6 +207,59 @@ response = await litellm.messages.acreate(
---
## Loop Ceiling
One intercepted request can chain several follow-up model calls, since the model often searches again after
reading the first set of results. `max_agentic_loops` caps how many of those follow-ups run, and it defaults
to 3. LiteLLM also breaks the loop early when the model asks for the exact same tool call twice in a row.
Set the ceiling on the feature, which the interceptor applies to `/v1/messages` requests:
```yaml
litellm_settings:
websearch_interception_params:
enabled_providers: ["bedrock"]
max_agentic_loops: 5
```
Or per deployment, which wins over the feature-level setting:
```yaml
model_list:
- model_name: claude-sonnet-4-5
litellm_params:
model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0
max_agentic_loops: 5
```
Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that
carries it is ignored and one request can never drive an unbounded number of upstream model calls.
Both places are validated at config load, and a value that is not an integer of at least 1 stops the proxy
from starting rather than surfacing later. The per-deployment one is checked while the model list is read,
not on `LiteLLM_Params`, because the proxy builds its router with `ignore_invalid_deployments=True` and a
validator down there would drop the deployment silently instead of refusing to start.
When the ceiling is reached on a non-streaming `/v1/messages` request, the turn ends there and the client gets
the last response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`.
The client never declared that tool, so leaving the block in would hand it a tool call it has no way to answer.
The answer can be less complete than it would have been with more loops, which is the tradeoff the ceiling
buys. Where the refused call was the only block left, the turn comes back with no text in it at all.
Non-streaming is not a limitation on the client here, because a client that asked for a stream gets the same
treatment. Interception converts an intercepted `stream=True` request to non-streaming before the loop runs and
rebuilds the SSE stream from the finalized turn afterwards, so the ceiling is always reached on a response the
client has not seen yet. `AgenticStreamingIterator` is the one caller that reaches the loop with its events
already on the wire, and it keeps raising, because a finalized turn would arrive there as a second message
rather than as a replacement.
Two other surfaces do not get that treatment yet. `/v1/responses` returns its own shape that the finalizer does
not rewrite, so it still hands back the internal call. And `/v1/chat/completions` runs its own copy of these
rails in `litellm_core_utils/chat_completion_agentic_loop.py`, which still raises rather than ending the turn.
Both are tracked separately
---
## Streaming Support
WebSearch interception works transparently with both streaming and non-streaming requests.

View file

@ -31,6 +31,9 @@ from litellm.integrations.websearch_interception.tools import (
from litellm.integrations.websearch_interception.transformation import (
WebSearchTransformation,
)
from litellm.litellm_core_utils.agentic_loop_settings import (
validated_max_agentic_loops,
)
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
@ -122,6 +125,7 @@ class WebSearchInterceptionLogger(CustomLogger):
self,
enabled_providers: list[LlmProviders | str] | None = None,
search_tool_name: str | None = None,
max_agentic_loops: int | None = None,
):
"""
Args:
@ -131,6 +135,9 @@ class WebSearchInterceptionLogger(CustomLogger):
Default: None (all providers enabled)
search_tool_name: Name of search tool configured in router's search_tools.
If None, will attempt to use first available search tool.
max_agentic_loops: How many follow-up model calls one intercepted request
may chain before the loop is refused and the turn ends.
If None, LiteLLM's default of 3 applies.
"""
super().__init__()
# Convert enum values to strings for comparison
@ -139,8 +146,16 @@ class WebSearchInterceptionLogger(CustomLogger):
else:
self.enabled_providers = [p.value if isinstance(p, LlmProviders) else p for p in enabled_providers]
self.search_tool_name = search_tool_name
self.max_agentic_loops = self._validated_max_agentic_loops(max_agentic_loops)
self._request_has_websearch = False # Track if current request has web search
@staticmethod
def _validated_max_agentic_loops(max_agentic_loops: object) -> int | None:
"""
Reject loop ceilings the agentic loop cannot honor, at config load time.
"""
return validated_max_agentic_loops(max_agentic_loops, field="websearch_interception_params.max_agentic_loops")
async def try_short_circuit_search(
self,
model: str,
@ -398,6 +413,7 @@ class WebSearchInterceptionLogger(CustomLogger):
websearch_interception_params:
enabled_providers: ["bedrock"]
search_tool_name: "my-perplexity-search"
max_agentic_loops: 5
Usage:
config = litellm_settings.get("websearch_interception_params", {})
@ -406,6 +422,7 @@ class WebSearchInterceptionLogger(CustomLogger):
# Extract parameters from config
enabled_providers_str: Final = config.get("enabled_providers", None)
search_tool_name: Final = config.get("search_tool_name", None)
max_agentic_loops: Final = config.get("max_agentic_loops", None)
# Convert string provider names to LlmProviders enum values
enabled_providers: list[LlmProviders | str] | None = None
@ -423,6 +440,7 @@ class WebSearchInterceptionLogger(CustomLogger):
return cls(
enabled_providers=enabled_providers,
search_tool_name=search_tool_name,
max_agentic_loops=max_agentic_loops,
)
@staticmethod
@ -493,6 +511,10 @@ class WebSearchInterceptionLogger(CustomLogger):
verbose_logger.debug("WebSearchInterception: Pre-request hook triggered for provider=%s", custom_llm_provider)
deployment_max_agentic_loops: Final = kwargs.get("max_agentic_loops")
if self.max_agentic_loops is not None and deployment_max_agentic_loops is None:
kwargs["max_agentic_loops"] = self.max_agentic_loops # rebind-ok: this hook returns the kwargs it edits
# If the client sent an Anthropic-native web_search_* tool, mark the
# request so the agentic loop emits native web_search_tool_result
# blocks in the final response (for citations panels, etc.). The flag

View file

@ -0,0 +1,59 @@
"""
Shared validation for the agentic loop ceiling.
``max_agentic_loops`` can be set in two places, and the two disagreed about
what a bad value means. The feature-level
``litellm_settings.websearch_interception_params.max_agentic_loops`` was
checked at config load, while a per-deployment
``model_list[].litellm_params.max_agentic_loops`` was passed straight through
to ``int(... or 3)``. That let a per-deployment ``0`` read as the default 3,
turning the tightest ceiling into the loosest one, and let a per-deployment
``"three"`` boot the proxy and then fail every request to that model.
Both settings now go through :func:`validated_max_agentic_loops`, which names
the field it rejected so the error says which line of the config to fix.
Anything that spells a whole number is still accepted, because the old
``int(... or 3)`` accepted those and a ceiling is routinely parameterized as
``max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS``, which resolves to a
string. Rejecting ``"5"`` would stop such a proxy from booting on upgrade.
"""
from typing import Final
DEFAULT_MAX_AGENTIC_LOOPS: Final = 3
def _as_whole_number(value: object) -> int | None:
"""
Return ``value`` as an int when it spells a whole number, else ``None``.
``bool`` is excluded explicitly because it is an ``int`` subclass, so
``max_agentic_loops: true`` would otherwise be read as a ceiling of 1.
"""
if isinstance(value, bool):
return None
if isinstance(value, int):
return value
if isinstance(value, float):
return int(value) if value.is_integer() else None
if isinstance(value, str):
try:
return int(value.strip())
except ValueError:
return None
return None
def validated_max_agentic_loops(max_agentic_loops: object, field: str) -> int | None:
"""
Return ``max_agentic_loops`` as an int, or raise naming ``field``.
"""
if max_agentic_loops is None:
return None
ceiling: Final = _as_whole_number(max_agentic_loops)
if ceiling is None:
raise TypeError(f"{field} must be an integer, got {max_agentic_loops!r}")
if ceiling < 1:
raise ValueError(f"{field} must be at least 1, got {ceiling}")
return ceiling

View file

@ -5,6 +5,10 @@ from typing import Final, cast
from litellm._logging import verbose_logger
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
)
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES,
@ -52,7 +56,10 @@ def _coerce_int(value: object, default: int) -> int:
def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]:
depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0)
max_loops: Final = max(_coerce_int(kwargs.get("max_agentic_loops"), 3), 1)
configured: Final = validated_max_agentic_loops(
kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
)
max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
raw_fingerprints: Final = kwargs.get("_agentic_loop_fingerprints")
fingerprints: Final = [str(fp) for fp in raw_fingerprints] if isinstance(raw_fingerprints, list) else []
return depth, max_loops, fingerprints

View file

@ -113,6 +113,14 @@ class FakeAnthropicMessagesStreamIterator:
}
chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode())
else:
passthrough_start: Final = {
"type": "content_block_start",
"index": index,
"content_block": block_dict,
}
chunks.append(f"event: content_block_start\ndata: {json.dumps(passthrough_start)}\n\n".encode())
content_block_stop: Final = {"type": "content_block_stop", "index": index}
chunks.append(f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode())
return chunks

View file

@ -19,6 +19,10 @@ import litellm.types.utils
from litellm._logging import _redact_string, verbose_logger
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
@ -89,6 +93,7 @@ from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadCon
from litellm.types.integrations.custom_logger import (
AgenticLoopPlan,
AgenticLoopRequestPatch,
AgenticLoopSafetyError,
)
from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
@ -5077,9 +5082,12 @@ class BaseLLMHTTPHandler:
@staticmethod
def _get_agentic_loop_settings(kwargs: dict) -> tuple[int, int, list[str]]:
depth: Final = int(kwargs.get("_agentic_loop_depth", 0) or 0)
max_loops: Final = int(kwargs.get("max_agentic_loops", 3) or 3)
configured: Final = validated_max_agentic_loops(
kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops"
)
max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured
fingerprints: Final = list(kwargs.get("_agentic_loop_fingerprints", []) or [])
return depth, max(max_loops, 1), fingerprints
return depth, max_loops, fingerprints
@staticmethod
def _has_agentic_completion_hook(logging_obj: LiteLLMLoggingObj) -> bool:
@ -5122,7 +5130,8 @@ class BaseLLMHTTPHandler:
"""
Evaluate agentic-loop safety guards (fingerprint cycle / max depth).
Raises ValueError on abort. Returns the current fingerprint on success.
Raises AgenticLoopSafetyError on abort. Returns the current fingerprint
on success.
These checks must not be swallowed by the per-callback ``except Exception``
block that wraps callback dispatch they are bounded-loop / cycle-break
@ -5130,9 +5139,9 @@ class BaseLLMHTTPHandler:
"""
fingerprint: Final = BaseLLMHTTPHandler._fingerprint_agentic_tools(tool_calls)
if fingerprint in fingerprints:
raise ValueError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun")
if depth >= max_loops:
raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}")
return fingerprint
@staticmethod
@ -5142,6 +5151,97 @@ class BaseLLMHTTPHandler:
except Exception:
return str(tools)
@staticmethod
def _refused_agentic_tool_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]:
"""
Collect the ids and names of the tool calls a safety rail just refused.
Callbacks hand back either a bare list of tool calls or a dict wrapping
that list under ``tool_calls``, and both the anthropic and responses
shapes carry an ``id`` (or ``call_id``) plus a ``name``.
"""
calls: Final = tool_calls.get("tool_calls") if isinstance(tool_calls, dict) else tool_calls
if not isinstance(calls, list):
return frozenset(), frozenset()
dict_calls: Final = (call for call in calls if isinstance(call, dict))
fields: Final = tuple((call.get("id"), call.get("call_id"), call.get("name")) for call in dict_calls)
ids: Final = frozenset(
value for call_id, caller_id, _ in fields for value in (call_id, caller_id) if isinstance(value, str)
)
names: Final = frozenset(name for _, _, name in fields if isinstance(name, str))
return ids, names
@staticmethod
def _is_refused_tool_use_block(block: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool:
"""
Whether this response block belongs to a tool call the rail refused.
An id settles it on its own, so a block carrying one is matched on the id
alone and a client's own tool call survives even where it happens to
share a name with a refused one. The name is only consulted for tool call
shapes that arrive without an id.
"""
if not isinstance(block, dict) or block.get("type") != "tool_use":
return False
block_id: Final = block.get("id")
if isinstance(block_id, str) and refused_ids:
return block_id in refused_ids
return block.get("name") in refused_names
@staticmethod
def _can_replace_turn_with_terminal_response(stream: bool, api_surface: str) -> bool:
"""
Whether a refused rerun can still be answered with a finalized turn.
Only the anthropic messages surface can. The responses surface carries a
pydantic model the finalizer does not rewrite, so it keeps raising, which
is what every surface did before this path learned to end the turn.
The messages and responses call sites pass ``stream=False``, because
interception converts an intercepted stream to non-streaming before the
loop runs and rebuilds the SSE stream from the finalized turn
afterwards. ``AgenticStreamingIterator`` passes ``stream=True``, and
that path keeps raising: its events are already on the wire, so a
finalized turn would reach the client as a second message rather than
as a replacement.
"""
return not stream and api_surface == "anthropic_messages"
@staticmethod
def _finalize_refused_agentic_response(response: object, tool_calls: object) -> object:
"""
Turn the response into a terminal turn after a safety rail refused the rerun.
The refused tool calls target tools LiteLLM injected on the client's
behalf, so a client that never declared them cannot send back a matching
``tool_result``. Their blocks are dropped and a ``tool_use`` stop reason
is closed out as ``end_turn``, which is what a provider-native web search
turn returns once it stops calling tools.
A ``tool_use`` block the client itself declared is left alone, and while
one is still in the response the stop reason stays ``tool_use`` so the
client knows to answer it.
"""
if not isinstance(response, dict):
return response
refused_ids, refused_names = BaseLLMHTTPHandler._refused_agentic_tool_identifiers(tool_calls)
finalized: Final = dict(response)
content: Final = finalized.get("content")
if isinstance(content, list):
kept_blocks: Final = [
block
for block in content
if not BaseLLMHTTPHandler._is_refused_tool_use_block(block, refused_ids, refused_names)
]
finalized["content"] = kept_blocks
client_tool_use_remains: Final = any(
isinstance(block, dict) and block.get("type") == "tool_use" for block in kept_blocks
)
if not client_tool_use_remains and finalized.get("stop_reason") == "tool_use":
finalized["stop_reason"] = "end_turn"
return finalized
async def _execute_anthropic_agentic_plan(
self,
plan: AgenticLoopPlan,
@ -5507,14 +5607,30 @@ class BaseLLMHTTPHandler:
continue
# Safety guards must run OUTSIDE the callback try/except — they are
# bounded-loop / cycle-break rails that must propagate to the caller.
fingerprint = self._check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
# bounded-loop / cycle-break rails, not callback bugs.
try:
fingerprint = self._check_agentic_loop_safety(
tool_calls=tool_calls,
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model=model,
)
except AgenticLoopSafetyError as e:
if not self._can_replace_turn_with_terminal_response(stream, api_surface):
raise
_call_id = getattr(logging_obj, "litellm_call_id", "unknown")
verbose_logger.warning(
"LiteLLM.AgenticLoopRefused: ending turn [call_id=%s model=%s]: %s",
_call_id,
model,
str(e),
)
return self._maybe_wrap_in_fake_stream(
self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls),
logging_obj,
api_surface,
)
try:
kwargs_with_provider = hook_kwargs.copy()

View file

@ -254,6 +254,9 @@ from litellm.exceptions import RejectedRequestError
from litellm.integrations.custom_guardrail import ModifyResponseException
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
from litellm.litellm_core_utils.agentic_loop_settings import (
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
@ -4081,6 +4084,27 @@ def resolve_complexity_router_plugins(
complexity_router_config["classifier_plugin"] = resolved_classifier # rebind-ok: out-param, resolved in place
def validate_deployment_max_agentic_loops(model: Mapping[str, Any]) -> None:
"""
Reject a per-deployment `max_agentic_loops` the agentic loop cannot honor.
Checked here rather than on `LiteLLM_Params` because the proxy builds its
router with `ignore_invalid_deployments=True`, so a validator down there
turns a bad value into a silently missing model instead of a refusal to
start. Left unchecked entirely, a `0` used to read as the default ceiling
of 3 and a non-integer failed every request to that model instead.
"""
litellm_params: Final = model.get("litellm_params") or {}
if "max_agentic_loops" not in litellm_params:
return
model_name: Final = model.get("model_name", "")
validated_max_agentic_loops(
litellm_params["max_agentic_loops"],
field=f"litellm_params.max_agentic_loops on model {model_name!r}",
)
def pin_complexity_router_model_id(model: dict) -> None: # mutable-ok: out-param, model_info is stamped in place
"""
Stamps `model_info.id` from the raw litellm_params before plugin resolution swaps
@ -5416,6 +5440,7 @@ class ProxyConfig:
for k, v in model["litellm_params"].items():
if isinstance(v, str) and v.startswith("os.environ/"):
model["litellm_params"][k] = get_secret(v)
validate_deployment_max_agentic_loops(model)
pin_complexity_router_model_id(model)
complexity_router_config = model["litellm_params"].get("complexity_router_config")
if isinstance(complexity_router_config, dict):

View file

@ -23,6 +23,21 @@ def is_interception_internal_key(
return any(key.startswith(prefix) for prefix in prefixes)
class AgenticLoopSafetyError(ValueError):
"""
Raised when an agentic-loop safety rail refuses a rerun.
Covers both rails: the bounded-loop cap (``max_agentic_loops``) and the
repeated tool-call fingerprint cycle break. Subclasses ``ValueError`` so
callers that already catch the broader type keep working.
Only the anthropic messages loop raises this today. The chat completions
loop in ``litellm_core_utils/chat_completion_agentic_loop.py`` still raises
a plain ``ValueError`` from its own copy of the same rails, so catching
this type alone will not cover that surface until it is moved over.
"""
class StandardCustomLoggerInitParams(BaseModel):
"""
Params for initializing a CustomLogger.

View file

@ -5,6 +5,7 @@ Type definitions for WebSearch Interception integration.
from typing import Literal, TypedDict
from pydantic import BaseModel
from typing_extensions import ReadOnly
class AnthropicSearchQuery(BaseModel):
@ -35,6 +36,7 @@ class WebSearchInterceptionConfig(TypedDict, total=False):
websearch_interception_params:
enabled_providers: ["bedrock"]
search_tool_name: "my-perplexity-search"
max_agentic_loops: 5
"""
enabled_providers: list[str]
@ -42,3 +44,6 @@ class WebSearchInterceptionConfig(TypedDict, total=False):
search_tool_name: str | None
"""Name of search tool configured in router's search_tools. If None, uses first available."""
max_agentic_loops: ReadOnly[int | None]
"""How many follow-up model calls one intercepted request may chain. If None, LiteLLM's default of 3 applies."""

View file

@ -0,0 +1,754 @@
"""
Unit tests for what an intercepted request returns once a safety rail refuses
another agentic loop.
The web search interception loop injects an internal tool (litellm_web_search)
that the client never declared. When the loop cap or the repeated-fingerprint
guard trips, the turn has to end with a terminal response: leaking that internal
tool_use block leaves the client holding a tool call it cannot answer.
Also covers the max_agentic_loops knob on websearch_interception_params, from
config.yaml through to the settings the loop actually reads.
"""
import json
from unittest.mock import MagicMock
import pytest
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.websearch_interception.handler import (
WebSearchInterceptionLogger,
)
from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
FakeAnthropicMessagesStreamIterator,
)
from litellm.litellm_core_utils.agentic_loop_settings import DEFAULT_MAX_AGENTIC_LOOPS
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.secret_managers.main import get_secret
from litellm.types.integrations.custom_logger import (
AgenticLoopPlan,
AgenticLoopRequestPatch,
AgenticLoopSafetyError,
)
INTERNAL_TOOL_NAME = "litellm_web_search"
@pytest.fixture(autouse=True)
def only_the_callbacks_these_tests_register(monkeypatch):
"""
These tests drive the hooks with a callback of their own on the logging
object, so a logger another test left on litellm.callbacks would join the
run and change what the hooks do.
"""
monkeypatch.setattr(litellm, "callbacks", [])
def _internal_tool_use_block(block_id: str = "toolu_internal_1") -> dict:
return {
"id": block_id,
"type": "tool_use",
"name": INTERNAL_TOOL_NAME,
"input": {"query": "who won the world cup"},
}
def _native_search_blocks(index: int = 1) -> list[dict]:
return [
{
"type": "server_tool_use",
"id": f"srvtoolu_{index}",
"name": "web_search",
"input": {"query": "who won the world cup"},
},
{
"type": "web_search_tool_result",
"tool_use_id": f"srvtoolu_{index}",
"content": [{"type": "web_search_result", "url": "https://example.com", "title": "Result"}],
},
]
def _response_asking_for_another_search(block_id: str = "toolu_internal_1") -> dict:
return {
"id": "msg_123",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [
*_native_search_blocks(index=1),
{"type": "text", "text": "Let me check one more source."},
_internal_tool_use_block(block_id),
],
"stop_reason": "tool_use",
"usage": {"input_tokens": 10, "output_tokens": 5},
}
def _block_types(response: dict) -> list[str]:
return [block["type"] for block in response["content"]]
def _tool_use_names(response: dict) -> list[str]:
return [block.get("name") for block in response["content"] if block.get("type") == "tool_use"]
class _InterceptingCallback(CustomLogger):
"""
Stands in for the websearch interceptor: asks for another loop whenever the
response carries an internal web search tool_use block, and injects the
native block pair on the way back out.
"""
def __init__(self):
self.plan_calls = 0
self.post_hook_calls = 0
async def async_should_run_agentic_loop(
self, response, model, messages, tools, stream, custom_llm_provider, kwargs
):
if not isinstance(response, dict):
return True, {"tool_calls": [_internal_tool_use_block()]}
tool_calls = [
block
for block in response.get("content", [])
if block.get("type") == "tool_use" and block.get("name") == INTERNAL_TOOL_NAME
]
if not tool_calls:
return False, {}
return True, {"tool_calls": tool_calls, "tool_type": "websearch"}
async def async_build_agentic_loop_plan(
self,
tools,
model,
messages,
response,
anthropic_messages_provider_config,
anthropic_messages_optional_request_params,
logging_obj,
stream,
kwargs,
):
self.plan_calls += 1
return AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(
messages=[{"role": "user", "content": "here are the search results"}],
max_tokens=1024,
),
)
async def async_post_agentic_loop_response_hook(self, response, plan, kwargs):
self.post_hook_calls += 1
if isinstance(response, dict):
response["content"] = [*_native_search_blocks(index=2), *response.get("content", [])]
return response
def _logging_obj(callback: CustomLogger, converted_stream: bool = False) -> MagicMock:
logging_obj = MagicMock()
logging_obj.model_call_details = {"websearch_interception_converted_stream": converted_stream}
logging_obj.dynamic_success_callbacks = [callback]
logging_obj.litellm_call_id = "call-abc"
return logging_obj
async def _run_hooks(
handler: BaseLLMHTTPHandler,
callback: CustomLogger,
kwargs: dict,
response: object = None,
stream: bool = False,
converted_stream: bool = False,
api_surface: str = "anthropic_messages",
):
return await handler._call_agentic_completion_hooks(
response=_response_asking_for_another_search() if response is None else response,
model="claude-sonnet-4-5",
messages=[{"role": "user", "content": "who won the world cup"}],
anthropic_messages_provider_config=MagicMock(),
anthropic_messages_optional_request_params={},
logging_obj=_logging_obj(callback, converted_stream=converted_stream),
stream=stream,
custom_llm_provider="anthropic",
kwargs=kwargs,
api_surface=api_surface,
)
class TestCappedLoopReturnsTerminalResponse:
def setup_method(self):
self.handler = BaseLLMHTTPHandler()
self.callback = _InterceptingCallback()
@pytest.mark.asyncio
async def test_internal_tool_use_block_is_dropped(self):
result = await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
)
assert isinstance(result, dict)
assert INTERNAL_TOOL_NAME not in _tool_use_names(result)
@pytest.mark.asyncio
async def test_stop_reason_is_closed_out(self):
result = await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
)
assert result["stop_reason"] == "end_turn"
@pytest.mark.asyncio
async def test_native_blocks_and_text_survive(self):
result = await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
)
assert _block_types(result) == ["server_tool_use", "web_search_tool_result", "text"]
@pytest.mark.asyncio
async def test_turn_carrying_only_the_refused_call_still_ends_cleanly(self):
"""
The refused call can be every block the model produced, which leaves the
turn with no content once it is dropped. That still has to come back as a
finished turn rather than as the leaked call, so the client stops instead
of waiting on a tool it cannot run, and the rest of the message survives
so the request is still billed and traceable.
An empty turn renders as nothing, which is the ceiling being set too low
for the question rather than a malformed response.
"""
nothing_but_the_refused_call = {
"id": "msg_123",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [_internal_tool_use_block()],
"stop_reason": "tool_use",
"usage": {"input_tokens": 10, "output_tokens": 5},
}
result = await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
response=nothing_but_the_refused_call,
)
assert result["content"] == []
assert result["stop_reason"] == "end_turn"
assert result["usage"] == {"input_tokens": 10, "output_tokens": 5}
assert result["id"] == "msg_123"
@pytest.mark.asyncio
async def test_no_follow_up_model_call_is_planned(self):
"""
The rail has to end the turn without planning another model call, and it
has to end it by returning rather than by raising, which is the half that
the caller's response depends on.
"""
result = await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
)
assert self.callback.plan_calls == 0
assert result["stop_reason"] == "end_turn"
@pytest.mark.asyncio
async def test_original_response_is_not_mutated(self):
response = _response_asking_for_another_search()
await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
response=response,
)
assert response["stop_reason"] == "tool_use"
assert INTERNAL_TOOL_NAME in _tool_use_names(response)
@pytest.mark.asyncio
async def test_repeated_fingerprint_guard_is_terminal_too(self):
tool_calls = {"tool_calls": [_internal_tool_use_block()], "tool_type": "websearch"}
seen = json.dumps(tool_calls, sort_keys=True, default=str)
result = await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, "_agentic_loop_fingerprints": [seen]},
)
assert self.callback.plan_calls == 0
assert INTERNAL_TOOL_NAME not in _tool_use_names(result)
assert result["stop_reason"] == "end_turn"
@pytest.mark.asyncio
async def test_client_declared_tool_use_is_left_alone(self):
response = _response_asking_for_another_search()
client_tool_use = {"id": "toolu_client_1", "type": "tool_use", "name": "get_weather", "input": {}}
response["content"].append(client_tool_use)
result = await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
response=response,
)
assert _tool_use_names(result) == ["get_weather"]
assert result["stop_reason"] == "tool_use"
def test_only_the_refused_tool_calls_are_dropped(self):
"""
A block is matched on the id the rail refused, not on the tool name, so a
second block sharing that name survives when the rail never listed it. A
callback that picks its tool calls out by name hands both over and both
go, which is its own call to make; this is about not widening it here.
"""
response = _response_asking_for_another_search()
response["content"].append(
{"id": "toolu_client_1", "type": "tool_use", "name": INTERNAL_TOOL_NAME, "input": {}}
)
result = BaseLLMHTTPHandler._finalize_refused_agentic_response(
response=response,
tool_calls={"tool_calls": [_internal_tool_use_block()]},
)
assert [block["id"] for block in result["content"] if block.get("type") == "tool_use"] == ["toolu_client_1"]
assert result["stop_reason"] == "tool_use"
def test_tool_calls_without_ids_still_match_by_name(self):
"""
Not every callback shape carries ids on its tool calls, so the name is
still what decides when the rail refused a call that has no id.
"""
result = BaseLLMHTTPHandler._finalize_refused_agentic_response(
response=_response_asking_for_another_search(),
tool_calls={"tool_calls": [{"name": INTERNAL_TOOL_NAME, "input": {}}]},
)
assert _tool_use_names(result) == []
assert result["stop_reason"] == "end_turn"
@pytest.mark.asyncio
async def test_streaming_caller_is_left_to_its_existing_behavior(self):
"""
A streaming caller has already sent the original message to the client, so
a finalized turn would land as a second message rather than replace the
first. The rail keeps raising there and the caller handles it as before.
"""
with pytest.raises(AgenticLoopSafetyError):
await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
stream=True,
)
assert self.callback.plan_calls == 0
@pytest.mark.asyncio
async def test_responses_surface_is_left_to_its_existing_behavior(self):
"""
The responses surface carries a pydantic model rather than the anthropic
dict this finalizer rewrites, so it keeps raising instead of being handed
a response that was never actually finalized.
"""
with pytest.raises(AgenticLoopSafetyError):
await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
api_surface="responses",
)
@pytest.mark.asyncio
async def test_non_dict_response_is_returned_untouched(self):
response = MagicMock()
result = await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
response=response,
)
assert result is response
@pytest.mark.asyncio
async def test_converted_stream_gets_a_terminal_fake_stream(self):
"""
A converted stream is wrapped back into an Anthropic SSE stream here, the
same as every other return in this function, so a streaming client gets a
terminal stream rather than a bare dict. The interceptor turns the client's
stream into a non-streaming upstream call, so stream is False on this path
and the converted flag on the logging object is what marks it.
"""
result = await _run_hooks(
self.handler,
self.callback,
kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3},
converted_stream=True,
)
assert isinstance(result, FakeAnthropicMessagesStreamIterator)
assert result.response["stop_reason"] == "end_turn"
assert INTERNAL_TOOL_NAME not in _tool_use_names(result.response)
def test_rails_cannot_trip_in_the_outermost_frame(self):
"""
Backs the invariant the test above relies on: at depth 0 the fingerprint set
is empty and the ceiling is at least 1, so neither rail can refuse.
"""
depth, max_loops, fingerprints = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={})
assert depth == 0
assert fingerprints == []
assert max_loops >= 1
depth, max_loops, fingerprints = BaseLLMHTTPHandler._get_agentic_loop_settings(
kwargs={"max_agentic_loops": 1}
)
assert max_loops == 1
assert BaseLLMHTTPHandler._check_agentic_loop_safety(
tool_calls={"tool_calls": [_internal_tool_use_block()]},
fingerprints=fingerprints,
depth=depth,
max_loops=max_loops,
model="claude-sonnet-4-5",
)
def test_safety_error_is_still_a_value_error(self):
assert issubclass(AgenticLoopSafetyError, ValueError)
def test_safety_error_type_names_the_rail(self):
with pytest.raises(AgenticLoopSafetyError, match="max_agentic_loops"):
BaseLLMHTTPHandler._check_agentic_loop_safety(
tool_calls={"tool_calls": [_internal_tool_use_block()]},
fingerprints=[],
depth=3,
max_loops=3,
model="claude-sonnet-4-5",
)
class TestOuterFramePostHookStillRuns:
"""
The cap used to raise through the parent frame's await, which skipped the
parent's post-loop hook. The parent now gets its terminal response back and
finishes normally, so the blocks it was going to inject still land.
"""
@pytest.mark.asyncio
async def test_parent_frame_injects_its_blocks_after_the_cap_trips(self, monkeypatch):
handler = BaseLLMHTTPHandler()
callback = _InterceptingCallback()
async def fake_acreate(**call_kwargs):
return await handler._call_agentic_completion_hooks(
response=_response_asking_for_another_search(block_id="toolu_internal_2"),
model=call_kwargs["model"],
messages=call_kwargs["messages"],
anthropic_messages_provider_config=MagicMock(),
anthropic_messages_optional_request_params={},
logging_obj=_logging_obj(callback),
stream=False,
custom_llm_provider="anthropic",
kwargs={
key: call_kwargs[key]
for key in ("_agentic_loop_depth", "max_agentic_loops", "_agentic_loop_fingerprints")
if key in call_kwargs
},
)
monkeypatch.setattr("litellm.anthropic_interface.messages.acreate", fake_acreate)
result = await _run_hooks(
handler,
callback,
kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 1},
)
assert callback.plan_calls == 1
assert callback.post_hook_calls == 1
assert _block_types(result)[:2] == ["server_tool_use", "web_search_tool_result"]
assert INTERNAL_TOOL_NAME not in _tool_use_names(result)
assert result["stop_reason"] == "end_turn"
class TestMaxAgenticLoopsConfigKnob:
def test_from_config_yaml_reads_the_knob(self):
logger = WebSearchInterceptionLogger.from_config_yaml(
{"enabled_providers": ["bedrock"], "max_agentic_loops": 7}
)
assert logger.max_agentic_loops == 7
def test_from_config_yaml_leaves_it_unset_by_default(self):
logger = WebSearchInterceptionLogger.from_config_yaml({"enabled_providers": ["bedrock"]})
assert logger.max_agentic_loops is None
@pytest.mark.parametrize("bad_value", [0, -1])
def test_out_of_range_ceilings_are_rejected_at_config_load(self, bad_value):
with pytest.raises(ValueError, match="max_agentic_loops"):
WebSearchInterceptionLogger.from_config_yaml(
{"enabled_providers": ["bedrock"], "max_agentic_loops": bad_value}
)
@pytest.mark.parametrize("bad_value", ["three", True, 2.5])
def test_non_integer_ceilings_are_rejected_at_config_load(self, bad_value):
with pytest.raises(TypeError, match="max_agentic_loops"):
WebSearchInterceptionLogger.from_config_yaml(
{"enabled_providers": ["bedrock"], "max_agentic_loops": bad_value}
)
def test_a_ceiling_spelled_as_a_string_is_read_at_config_load(self):
"""
`max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS` resolves to a string
before it reaches the knob, so refusing "5" would break a config that
works today.
"""
logger = WebSearchInterceptionLogger.from_config_yaml(
{"enabled_providers": ["bedrock"], "max_agentic_loops": "5"}
)
assert logger.max_agentic_loops == 5
@pytest.mark.asyncio
async def test_knob_reaches_the_loop_settings(self):
logger = WebSearchInterceptionLogger.from_config_yaml(
{"enabled_providers": ["bedrock"], "max_agentic_loops": 7}
)
kwargs = {
"tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}],
"litellm_params": {"custom_llm_provider": "bedrock"},
}
updated = await logger.async_pre_request_hook(model="claude-sonnet-4-5", messages=[], kwargs=kwargs)
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs=updated)
assert max_loops == 7
@pytest.mark.asyncio
async def test_deployment_setting_wins_over_the_feature_setting(self):
logger = WebSearchInterceptionLogger.from_config_yaml(
{"enabled_providers": ["bedrock"], "max_agentic_loops": 7}
)
kwargs = {
"tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}],
"litellm_params": {"custom_llm_provider": "bedrock"},
"max_agentic_loops": 2,
}
updated = await logger.async_pre_request_hook(model="claude-sonnet-4-5", messages=[], kwargs=kwargs)
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs=updated)
assert max_loops == 2
@pytest.mark.asyncio
async def test_default_ceiling_applies_when_the_knob_is_unset(self):
logger = WebSearchInterceptionLogger.from_config_yaml({"enabled_providers": ["bedrock"]})
kwargs = {
"tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}],
"litellm_params": {"custom_llm_provider": "bedrock"},
}
updated = await logger.async_pre_request_hook(model="claude-sonnet-4-5", messages=[], kwargs=kwargs)
assert "max_agentic_loops" not in updated
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs=updated)
assert max_loops == 3
def _stream_events(response: dict) -> list[dict]:
events: list[dict] = []
for chunk in FakeAnthropicMessagesStreamIterator(response=response):
for line in chunk.decode().splitlines():
if line.startswith("data: "):
events.append(json.loads(line[len("data: ") :]))
return events
class TestBothCeilingKnobsAreValidated:
"""
``max_agentic_loops`` is settable per deployment and feature-wide, and the
per-deployment one wins. Only the feature-wide one used to be checked, so a
per-deployment ``0`` was swallowed by an ``or 3`` and read as the default 3,
handing the loosest ceiling to whoever asked for the tightest.
"""
def test_a_per_deployment_zero_is_rejected_not_read_as_the_default(self):
with pytest.raises(ValueError, match="must be at least 1, got 0"):
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 0})
def test_a_per_deployment_non_integer_names_the_field_it_came_from(self):
with pytest.raises(TypeError, match=r"litellm_params\.max_agentic_loops must be an integer"):
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": "three"})
def test_a_per_deployment_true_is_not_read_as_a_ceiling_of_one(self):
with pytest.raises(TypeError, match="must be an integer"):
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": True})
def test_an_absent_ceiling_falls_back_to_the_shared_default(self):
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={})
assert max_loops == DEFAULT_MAX_AGENTIC_LOOPS
def test_an_explicit_none_falls_back_to_the_shared_default(self):
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": None})
assert max_loops == DEFAULT_MAX_AGENTIC_LOOPS
def test_a_valid_per_deployment_ceiling_is_passed_through(self):
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 6})
assert max_loops == 6
@pytest.mark.parametrize("rejected", [0, -1, "three", True])
def test_the_two_knobs_reject_the_same_values(self, rejected):
with pytest.raises((TypeError, ValueError)):
WebSearchInterceptionLogger(max_agentic_loops=rejected)
with pytest.raises((TypeError, ValueError)):
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": rejected})
def test_each_knob_names_its_own_config_field(self):
with pytest.raises(ValueError, match=r"websearch_interception_params\.max_agentic_loops"):
WebSearchInterceptionLogger(max_agentic_loops=0)
with pytest.raises(ValueError, match=r"litellm_params\.max_agentic_loops"):
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 0})
class TestACeilingThatSpellsAWholeNumberStillWorks:
"""
The ceiling used to go through ``int(... or 3)``, which accepted anything
``int()`` accepted. A ceiling is routinely parameterized as
``max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS``, and ``get_secret``
hands that back as the string ``"5"``, so tightening the check to
``isinstance(int)`` would stop such a proxy from booting on upgrade.
"""
@pytest.mark.parametrize("spelled", ["5", " 5 ", 5.0])
def test_a_ceiling_that_spells_five_is_accepted_by_both_knobs(self, spelled):
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": spelled})
assert max_loops == 5
assert WebSearchInterceptionLogger(max_agentic_loops=spelled).max_agentic_loops == 5
def test_an_env_var_sourced_ceiling_survives_secret_resolution(self, monkeypatch):
monkeypatch.setenv("MAX_AGENTIC_LOOPS_UNDER_TEST", "7")
resolved = get_secret("os.environ/MAX_AGENTIC_LOOPS_UNDER_TEST")
assert isinstance(resolved, str)
_, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": resolved})
assert max_loops == 7
def test_a_spelled_zero_is_still_refused_and_reports_the_number(self):
with pytest.raises(ValueError, match="must be at least 1, got 0"):
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": "0"})
def test_a_word_is_still_refused(self):
with pytest.raises(TypeError, match=r"litellm_params\.max_agentic_loops must be an integer"):
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": "three"})
def test_a_fractional_ceiling_is_refused_rather_than_truncated(self):
with pytest.raises(TypeError, match="must be an integer"):
BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 5.5})
class TestRebuiltStreamIsWellFormed:
"""
A capped turn is rebuilt into SSE by FakeAnthropicMessagesStreamIterator.
Anthropic's SDK accumulator appends on content_block_start and then indexes
content[event.index] on content_block_delta, so a block that stops without
ever starting shifts every later index and the accumulator raises
IndexError. A web search turn carries server_tool_use and
web_search_tool_result blocks, which is exactly where that used to happen.
"""
@staticmethod
def _capped_search_turn() -> dict:
return {
"id": "msg_01",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"stop_reason": "end_turn",
"content": [
{
"type": "server_tool_use",
"id": "srvtoolu_01",
"name": "web_search",
"input": {"query": "on-demand H100 hourly price"},
},
{
"type": "web_search_tool_result",
"tool_use_id": "srvtoolu_01",
"content": [
{
"type": "web_search_result",
"url": "https://example.com/h100",
"title": "H100 pricing",
}
],
},
{"type": "text", "text": "AWS lists the H100 at $12.29 an hour."},
],
"usage": {"input_tokens": 100, "output_tokens": 20},
}
def test_every_content_block_stop_has_a_matching_start(self):
events = _stream_events(self._capped_search_turn())
started = [event["index"] for event in events if event["type"] == "content_block_start"]
stopped = [event["index"] for event in events if event["type"] == "content_block_stop"]
assert started == [0, 1, 2]
assert stopped == [0, 1, 2]
def test_no_delta_indexes_past_the_blocks_started_before_it(self):
events = _stream_events(self._capped_search_turn())
blocks_started = 0
for event in events:
if event["type"] == "content_block_start":
blocks_started += 1
elif event["type"] == "content_block_delta":
assert event["index"] < blocks_started
def test_search_blocks_reach_the_client(self):
events = _stream_events(self._capped_search_turn())
started_types = [
event["content_block"]["type"] for event in events if event["type"] == "content_block_start"
]
assert started_types == ["server_tool_use", "web_search_tool_result", "text"]
def test_the_search_result_survives_the_rebuild_intact(self):
events = _stream_events(self._capped_search_turn())
result_block = next(
event["content_block"]
for event in events
if event["type"] == "content_block_start"
and event["content_block"]["type"] == "web_search_tool_result"
)
assert result_block["tool_use_id"] == "srvtoolu_01"
assert result_block["content"][0]["url"] == "https://example.com/h100"

View file

@ -26,6 +26,7 @@ from litellm.proxy.proxy_server import (
_scrub_guardrail_inner,
resolve_complexity_router_plugins,
resolve_routing_plugins,
validate_deployment_max_agentic_loops,
)
from .conftest import normalize
@ -153,6 +154,71 @@ def test_resolve_complexity_router_plugins_resolves_dotted_path_to_live_instance
assert type(config["plugins"][0]).__name__ == "_Plugin"
def test_validate_deployment_max_agentic_loops_allows_a_deployment_without_the_key():
model = {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}}
validate_deployment_max_agentic_loops(model)
assert "max_agentic_loops" not in model["litellm_params"]
def test_validate_deployment_max_agentic_loops_leaves_a_valid_ceiling_alone():
model = {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": 5}}
validate_deployment_max_agentic_loops(model)
assert model["litellm_params"]["max_agentic_loops"] == 5
def test_validate_deployment_max_agentic_loops_rejects_zero():
"""
A per-deployment 0 used to be swallowed by an `or 3` and read as the default
ceiling of 3, handing the loosest setting to whoever asked for the tightest.
"""
with pytest.raises(ValueError, match="must be at least 1, got 0"):
validate_deployment_max_agentic_loops(
{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": 0}}
)
def test_validate_deployment_max_agentic_loops_rejects_a_non_integer():
"""
A per-deployment non-integer used to let the proxy boot and then fail every
request to that model with `invalid literal for int() with base 10`.
"""
with pytest.raises(TypeError, match="must be an integer"):
validate_deployment_max_agentic_loops(
{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": "three"}}
)
def test_validate_deployment_max_agentic_loops_rejects_a_bool():
with pytest.raises(TypeError, match="must be an integer"):
validate_deployment_max_agentic_loops(
{"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": True}}
)
def test_validate_deployment_max_agentic_loops_accepts_a_ceiling_from_an_env_var():
"""
`max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS` is resolved to a string
before this check runs, and the old `int(... or 3)` accepted that, so
refusing it here would stop an already working proxy from booting.
"""
model = {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": "5"}}
validate_deployment_max_agentic_loops(model)
assert model["litellm_params"]["max_agentic_loops"] == "5"
def test_validate_deployment_max_agentic_loops_names_the_offending_model():
with pytest.raises(ValueError, match="on model 'claude-sonnet-4-5'"):
validate_deployment_max_agentic_loops(
{"model_name": "claude-sonnet-4-5", "litellm_params": {"max_agentic_loops": -1}}
)
def test_resolve_complexity_router_plugins_rejects_non_routing_plugin_object(tmp_path):
plugin_file = tmp_path / "bad_plugin.py"
plugin_file.write_text("not_a_plugin = object()\n")