From 23a9300da6e97dc7a37816b027050508cd4237af Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:19:42 -0700 Subject: [PATCH 1/8] fix(websearch_interception): end the turn when the agentic loop hits its ceiling When the bounded loop cap or the repeated tool-call fingerprint guard refused a rerun, the raise escaped the parent agentic frame and the client got the raw model turn back: HTTP 200 carrying an unresolved tool_use block for the internal litellm_web_search tool and stop_reason "tool_use". The client never declared that tool, so it had no way to answer it and the conversation could not continue The safety check now raises AgenticLoopSafetyError, a ValueError subclass, and _call_agentic_completion_hooks catches it and returns a finalized response: the blocks belonging to the refused tool calls are dropped, and stop_reason is closed out to end_turn when nothing the client declared is still waiting. Refused blocks are matched by the ids and names of the tool calls the rail refused rather than by hardcoding the web search tool name Only the non-streaming anthropic messages path ends the turn this way. A streaming caller has already sent the original message by the time the hooks run, so a finalized turn would arrive as a second message rather than replace the first, and the responses surface carries a pydantic model this finalizer does not rewrite. Both keep raising, exactly as they did before Also adds max_agentic_loops to websearch_interception_params so the ceiling can be set once for the whole feature. A per deployment litellm_params.max_agentic_loops still wins over it, and the field stays on the proxy's untrusted root list so a client cannot raise its own ceiling --- .../websearch_interception/ARCHITECTURE.md | 35 ++ .../websearch_interception/handler.py | 32 ++ litellm/llms/custom_httpx/llm_http_handler.py | 126 ++++- litellm/types/integrations/custom_logger.py | 10 + .../integrations/websearch_interception.py | 5 + .../test_websearch_agentic_loop_cap.py | 527 ++++++++++++++++++ 6 files changed, 724 insertions(+), 11 deletions(-) create mode 100644 tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py diff --git a/litellm/integrations/websearch_interception/ARCHITECTURE.md b/litellm/integrations/websearch_interception/ARCHITECTURE.md index ce7f01c5a2a..691bb26880e 100644 --- a/litellm/integrations/websearch_interception/ARCHITECTURE.md +++ b/litellm/integrations/websearch_interception/ARCHITECTURE.md @@ -207,6 +207,41 @@ response = await litellm.messages.acreate( --- +## Loop Ceiling + +One intercepted request can chain several follow-up model calls, since the model often searches again after +reading the first set of results. `max_agentic_loops` caps how many of those follow-ups run, and it defaults +to 3. LiteLLM also breaks the loop early when the model asks for the exact same tool call twice in a row. + +Set the ceiling on the feature, which the interceptor applies to `/v1/messages` requests: + +```yaml +litellm_settings: + websearch_interception_params: + enabled_providers: ["bedrock"] + max_agentic_loops: 5 +``` + +Or per deployment, which wins over the feature-level setting: + +```yaml +model_list: + - model_name: claude-sonnet-4-5 + litellm_params: + model: bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0 + max_agentic_loops: 5 +``` + +Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that +carries it is ignored and one request can never drive an unbounded number of upstream model calls. + +When the ceiling is reached, the turn ends there and the client gets the last response back with the internal +`litellm_web_search` tool call removed and `stop_reason: end_turn`. The client never declared that tool, so +leaving the block in would hand it a tool call it has no way to answer. The answer can be less complete than +it would have been with more loops, which is the tradeoff the ceiling buys + +--- + ## Streaming Support WebSearch interception works transparently with both streaming and non-streaming requests. diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index e59ef0449d0..760824f820f 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -122,6 +122,7 @@ class WebSearchInterceptionLogger(CustomLogger): self, enabled_providers: list[LlmProviders | str] | None = None, search_tool_name: str | None = None, + max_agentic_loops: int | None = None, ): """ Args: @@ -131,6 +132,9 @@ class WebSearchInterceptionLogger(CustomLogger): Default: None (all providers enabled) search_tool_name: Name of search tool configured in router's search_tools. If None, will attempt to use first available search tool. + max_agentic_loops: How many follow-up model calls one intercepted request + may chain before the loop is refused and the turn ends. + If None, LiteLLM's default of 3 applies. """ super().__init__() # Convert enum values to strings for comparison @@ -139,8 +143,29 @@ class WebSearchInterceptionLogger(CustomLogger): else: self.enabled_providers = [p.value if isinstance(p, LlmProviders) else p for p in enabled_providers] self.search_tool_name = search_tool_name + self.max_agentic_loops = self._validated_max_agentic_loops(max_agentic_loops) self._request_has_websearch = False # Track if current request has web search + @staticmethod + def _validated_max_agentic_loops(max_agentic_loops: object) -> int | None: + """ + Reject loop ceilings the agentic loop cannot honor, at config load time. + + ``bool`` is excluded explicitly because it is an ``int`` subclass, so + ``max_agentic_loops: true`` would otherwise be read as a ceiling of 1. + """ + if max_agentic_loops is None: + return None + if isinstance(max_agentic_loops, bool) or not isinstance(max_agentic_loops, int): + raise TypeError( + f"websearch_interception_params.max_agentic_loops must be an integer, got {max_agentic_loops!r}" + ) + if max_agentic_loops < 1: + raise ValueError( + f"websearch_interception_params.max_agentic_loops must be at least 1, got {max_agentic_loops}" + ) + return max_agentic_loops + async def try_short_circuit_search( self, model: str, @@ -398,6 +423,7 @@ class WebSearchInterceptionLogger(CustomLogger): websearch_interception_params: enabled_providers: ["bedrock"] search_tool_name: "my-perplexity-search" + max_agentic_loops: 5 Usage: config = litellm_settings.get("websearch_interception_params", {}) @@ -406,6 +432,7 @@ class WebSearchInterceptionLogger(CustomLogger): # Extract parameters from config enabled_providers_str: Final = config.get("enabled_providers", None) search_tool_name: Final = config.get("search_tool_name", None) + max_agentic_loops: Final = config.get("max_agentic_loops", None) # Convert string provider names to LlmProviders enum values enabled_providers: list[LlmProviders | str] | None = None @@ -423,6 +450,7 @@ class WebSearchInterceptionLogger(CustomLogger): return cls( enabled_providers=enabled_providers, search_tool_name=search_tool_name, + max_agentic_loops=max_agentic_loops, ) @staticmethod @@ -493,6 +521,10 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.debug("WebSearchInterception: Pre-request hook triggered for provider=%s", custom_llm_provider) + deployment_max_agentic_loops: Final = kwargs.get("max_agentic_loops") + if self.max_agentic_loops is not None and deployment_max_agentic_loops is None: + kwargs["max_agentic_loops"] = self.max_agentic_loops # rebind-ok: this hook returns the kwargs it edits + # If the client sent an Anthropic-native web_search_* tool, mark the # request so the agentic loop emits native web_search_tool_result # blocks in the final response (for citations panels, etc.). The flag diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 8c98c526da1..a14a89613c6 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -89,6 +89,7 @@ from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadCon from litellm.types.integrations.custom_logger import ( AgenticLoopPlan, AgenticLoopRequestPatch, + AgenticLoopSafetyError, ) from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, @@ -5122,7 +5123,8 @@ class BaseLLMHTTPHandler: """ Evaluate agentic-loop safety guards (fingerprint cycle / max depth). - Raises ValueError on abort. Returns the current fingerprint on success. + Raises AgenticLoopSafetyError on abort. Returns the current fingerprint + on success. These checks must not be swallowed by the per-callback ``except Exception`` block that wraps callback dispatch — they are bounded-loop / cycle-break @@ -5130,9 +5132,9 @@ class BaseLLMHTTPHandler: """ fingerprint: Final = BaseLLMHTTPHandler._fingerprint_agentic_tools(tool_calls) if fingerprint in fingerprints: - raise ValueError("Agentic loop detected repeated tool-call fingerprint; aborting rerun") + raise AgenticLoopSafetyError("Agentic loop detected repeated tool-call fingerprint; aborting rerun") if depth >= max_loops: - raise ValueError(f"Exceeded max_agentic_loops={max_loops} for model={model}") + raise AgenticLoopSafetyError(f"Exceeded max_agentic_loops={max_loops} for model={model}") return fingerprint @staticmethod @@ -5142,6 +5144,92 @@ class BaseLLMHTTPHandler: except Exception: return str(tools) + @staticmethod + def _refused_agentic_tool_identifiers(tool_calls: object) -> tuple[frozenset[str], frozenset[str]]: + """ + Collect the ids and names of the tool calls a safety rail just refused. + + Callbacks hand back either a bare list of tool calls or a dict wrapping + that list under ``tool_calls``, and both the anthropic and responses + shapes carry an ``id`` (or ``call_id``) plus a ``name``. + """ + calls: Final = tool_calls.get("tool_calls") if isinstance(tool_calls, dict) else tool_calls + if not isinstance(calls, list): + return frozenset(), frozenset() + dict_calls: Final = (call for call in calls if isinstance(call, dict)) + fields: Final = tuple((call.get("id"), call.get("call_id"), call.get("name")) for call in dict_calls) + ids: Final = frozenset( + value for call_id, caller_id, _ in fields for value in (call_id, caller_id) if isinstance(value, str) + ) + names: Final = frozenset(name for _, _, name in fields if isinstance(name, str)) + return ids, names + + @staticmethod + def _is_refused_tool_use_block(block: object, refused_ids: frozenset[str], refused_names: frozenset[str]) -> bool: + """ + Whether this response block belongs to a tool call the rail refused. + + An id settles it on its own, so a block carrying one is matched on the id + alone and a client's own tool call survives even where it happens to + share a name with a refused one. The name is only consulted for tool call + shapes that arrive without an id. + """ + if not isinstance(block, dict) or block.get("type") != "tool_use": + return False + block_id: Final = block.get("id") + if isinstance(block_id, str) and refused_ids: + return block_id in refused_ids + return block.get("name") in refused_names + + @staticmethod + def _can_replace_turn_with_terminal_response(stream: bool, api_surface: str) -> bool: + """ + Whether a refused rerun can still be answered with a finalized turn. + + Only the non-streaming anthropic messages path can. A streaming caller + has already sent the original message to the client, so a finalized one + would arrive as a second message rather than as a replacement, and the + responses surface carries a pydantic model that the finalizer does not + rewrite. Both keep raising, which is what every surface did before this + path learned to end the turn. + """ + return not stream and api_surface == "anthropic_messages" + + @staticmethod + def _finalize_refused_agentic_response(response: object, tool_calls: object) -> object: + """ + Turn the response into a terminal turn after a safety rail refused the rerun. + + The refused tool calls target tools LiteLLM injected on the client's + behalf, so a client that never declared them cannot send back a matching + ``tool_result``. Their blocks are dropped and a ``tool_use`` stop reason + is closed out as ``end_turn``, which is what a provider-native web search + turn returns once it stops calling tools. + + A ``tool_use`` block the client itself declared is left alone, and while + one is still in the response the stop reason stays ``tool_use`` so the + client knows to answer it. + """ + if not isinstance(response, dict): + return response + + refused_ids, refused_names = BaseLLMHTTPHandler._refused_agentic_tool_identifiers(tool_calls) + finalized: Final = dict(response) + content: Final = finalized.get("content") + if isinstance(content, list): + kept_blocks: Final = [ + block + for block in content + if not BaseLLMHTTPHandler._is_refused_tool_use_block(block, refused_ids, refused_names) + ] + finalized["content"] = kept_blocks + client_tool_use_remains: Final = any( + isinstance(block, dict) and block.get("type") == "tool_use" for block in kept_blocks + ) + if not client_tool_use_remains and finalized.get("stop_reason") == "tool_use": + finalized["stop_reason"] = "end_turn" + return finalized + async def _execute_anthropic_agentic_plan( self, plan: AgenticLoopPlan, @@ -5507,14 +5595,30 @@ class BaseLLMHTTPHandler: continue # Safety guards must run OUTSIDE the callback try/except — they are - # bounded-loop / cycle-break rails that must propagate to the caller. - fingerprint = self._check_agentic_loop_safety( - tool_calls=tool_calls, - fingerprints=fingerprints, - depth=depth, - max_loops=max_loops, - model=model, - ) + # bounded-loop / cycle-break rails, not callback bugs. + try: + fingerprint = self._check_agentic_loop_safety( + tool_calls=tool_calls, + fingerprints=fingerprints, + depth=depth, + max_loops=max_loops, + model=model, + ) + except AgenticLoopSafetyError as e: + if not self._can_replace_turn_with_terminal_response(stream, api_surface): + raise + _call_id = getattr(logging_obj, "litellm_call_id", "unknown") + verbose_logger.warning( + "LiteLLM.AgenticLoopRefused: ending turn [call_id=%s model=%s]: %s", + _call_id, + model, + str(e), + ) + return self._maybe_wrap_in_fake_stream( + self._finalize_refused_agentic_response(response=response, tool_calls=tool_calls), + logging_obj, + api_surface, + ) try: kwargs_with_provider = hook_kwargs.copy() diff --git a/litellm/types/integrations/custom_logger.py b/litellm/types/integrations/custom_logger.py index 89b85bc5114..6b1bb2f449f 100644 --- a/litellm/types/integrations/custom_logger.py +++ b/litellm/types/integrations/custom_logger.py @@ -23,6 +23,16 @@ def is_interception_internal_key( return any(key.startswith(prefix) for prefix in prefixes) +class AgenticLoopSafetyError(ValueError): + """ + Raised when an agentic-loop safety rail refuses a rerun. + + Covers both rails: the bounded-loop cap (``max_agentic_loops``) and the + repeated tool-call fingerprint cycle break. Subclasses ``ValueError`` so + callers that already catch the broader type keep working. + """ + + class StandardCustomLoggerInitParams(BaseModel): """ Params for initializing a CustomLogger. diff --git a/litellm/types/integrations/websearch_interception.py b/litellm/types/integrations/websearch_interception.py index 90713b270be..7926b9eee0a 100644 --- a/litellm/types/integrations/websearch_interception.py +++ b/litellm/types/integrations/websearch_interception.py @@ -5,6 +5,7 @@ Type definitions for WebSearch Interception integration. from typing import Literal, TypedDict from pydantic import BaseModel +from typing_extensions import ReadOnly class AnthropicSearchQuery(BaseModel): @@ -35,6 +36,7 @@ class WebSearchInterceptionConfig(TypedDict, total=False): websearch_interception_params: enabled_providers: ["bedrock"] search_tool_name: "my-perplexity-search" + max_agentic_loops: 5 """ enabled_providers: list[str] @@ -42,3 +44,6 @@ class WebSearchInterceptionConfig(TypedDict, total=False): search_tool_name: str | None """Name of search tool configured in router's search_tools. If None, uses first available.""" + + max_agentic_loops: ReadOnly[int | None] + """How many follow-up model calls one intercepted request may chain. If None, LiteLLM's default of 3 applies.""" diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py new file mode 100644 index 00000000000..1d36ca76832 --- /dev/null +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py @@ -0,0 +1,527 @@ +""" +Unit tests for what an intercepted request returns once a safety rail refuses +another agentic loop. + +The web search interception loop injects an internal tool (litellm_web_search) +that the client never declared. When the loop cap or the repeated-fingerprint +guard trips, the turn has to end with a terminal response: leaking that internal +tool_use block leaves the client holding a tool call it cannot answer. + +Also covers the max_agentic_loops knob on websearch_interception_params, from +config.yaml through to the settings the loop actually reads. +""" + +import json +from unittest.mock import MagicMock + +import pytest + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.websearch_interception.handler import ( + WebSearchInterceptionLogger, +) +from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( + FakeAnthropicMessagesStreamIterator, +) +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.types.integrations.custom_logger import ( + AgenticLoopPlan, + AgenticLoopRequestPatch, + AgenticLoopSafetyError, +) + +INTERNAL_TOOL_NAME = "litellm_web_search" + + +@pytest.fixture(autouse=True) +def only_the_callbacks_these_tests_register(monkeypatch): + """ + These tests drive the hooks with a callback of their own on the logging + object, so a logger another test left on litellm.callbacks would join the + run and change what the hooks do. + """ + monkeypatch.setattr(litellm, "callbacks", []) + + +def _internal_tool_use_block(block_id: str = "toolu_internal_1") -> dict: + return { + "id": block_id, + "type": "tool_use", + "name": INTERNAL_TOOL_NAME, + "input": {"query": "who won the world cup"}, + } + + +def _native_search_blocks(index: int = 1) -> list[dict]: + return [ + { + "type": "server_tool_use", + "id": f"srvtoolu_{index}", + "name": "web_search", + "input": {"query": "who won the world cup"}, + }, + { + "type": "web_search_tool_result", + "tool_use_id": f"srvtoolu_{index}", + "content": [{"type": "web_search_result", "url": "https://example.com", "title": "Result"}], + }, + ] + + +def _response_asking_for_another_search(block_id: str = "toolu_internal_1") -> dict: + return { + "id": "msg_123", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [ + *_native_search_blocks(index=1), + {"type": "text", "text": "Let me check one more source."}, + _internal_tool_use_block(block_id), + ], + "stop_reason": "tool_use", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + + +def _block_types(response: dict) -> list[str]: + return [block["type"] for block in response["content"]] + + +def _tool_use_names(response: dict) -> list[str]: + return [block.get("name") for block in response["content"] if block.get("type") == "tool_use"] + + +class _InterceptingCallback(CustomLogger): + """ + Stands in for the websearch interceptor: asks for another loop whenever the + response carries an internal web search tool_use block, and injects the + native block pair on the way back out. + """ + + def __init__(self): + self.plan_calls = 0 + self.post_hook_calls = 0 + + async def async_should_run_agentic_loop( + self, response, model, messages, tools, stream, custom_llm_provider, kwargs + ): + if not isinstance(response, dict): + return True, {"tool_calls": [_internal_tool_use_block()]} + tool_calls = [ + block + for block in response.get("content", []) + if block.get("type") == "tool_use" and block.get("name") == INTERNAL_TOOL_NAME + ] + if not tool_calls: + return False, {} + return True, {"tool_calls": tool_calls, "tool_type": "websearch"} + + async def async_build_agentic_loop_plan( + self, + tools, + model, + messages, + response, + anthropic_messages_provider_config, + anthropic_messages_optional_request_params, + logging_obj, + stream, + kwargs, + ): + self.plan_calls += 1 + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch( + messages=[{"role": "user", "content": "here are the search results"}], + max_tokens=1024, + ), + ) + + async def async_post_agentic_loop_response_hook(self, response, plan, kwargs): + self.post_hook_calls += 1 + if isinstance(response, dict): + response["content"] = [*_native_search_blocks(index=2), *response.get("content", [])] + return response + + +def _logging_obj(callback: CustomLogger, converted_stream: bool = False) -> MagicMock: + logging_obj = MagicMock() + logging_obj.model_call_details = {"websearch_interception_converted_stream": converted_stream} + logging_obj.dynamic_success_callbacks = [callback] + logging_obj.litellm_call_id = "call-abc" + return logging_obj + + +async def _run_hooks( + handler: BaseLLMHTTPHandler, + callback: CustomLogger, + kwargs: dict, + response: object = None, + stream: bool = False, + converted_stream: bool = False, + api_surface: str = "anthropic_messages", +): + return await handler._call_agentic_completion_hooks( + response=_response_asking_for_another_search() if response is None else response, + model="claude-sonnet-4-5", + messages=[{"role": "user", "content": "who won the world cup"}], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={}, + logging_obj=_logging_obj(callback, converted_stream=converted_stream), + stream=stream, + custom_llm_provider="anthropic", + kwargs=kwargs, + api_surface=api_surface, + ) + + +class TestCappedLoopReturnsTerminalResponse: + def setup_method(self): + self.handler = BaseLLMHTTPHandler() + self.callback = _InterceptingCallback() + + @pytest.mark.asyncio + async def test_internal_tool_use_block_is_dropped(self): + result = await _run_hooks( + self.handler, + self.callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + ) + + assert isinstance(result, dict) + assert INTERNAL_TOOL_NAME not in _tool_use_names(result) + + @pytest.mark.asyncio + async def test_stop_reason_is_closed_out(self): + result = await _run_hooks( + self.handler, + self.callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + ) + + assert result["stop_reason"] == "end_turn" + + @pytest.mark.asyncio + async def test_native_blocks_and_text_survive(self): + result = await _run_hooks( + self.handler, + self.callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + ) + + assert _block_types(result) == ["server_tool_use", "web_search_tool_result", "text"] + + @pytest.mark.asyncio + async def test_no_follow_up_model_call_is_planned(self): + """ + The rail has to end the turn without planning another model call, and it + has to end it by returning rather than by raising, which is the half that + the caller's response depends on. + """ + result = await _run_hooks( + self.handler, + self.callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + ) + + assert self.callback.plan_calls == 0 + assert result["stop_reason"] == "end_turn" + + @pytest.mark.asyncio + async def test_original_response_is_not_mutated(self): + response = _response_asking_for_another_search() + + await _run_hooks( + self.handler, + self.callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + response=response, + ) + + assert response["stop_reason"] == "tool_use" + assert INTERNAL_TOOL_NAME in _tool_use_names(response) + + @pytest.mark.asyncio + async def test_repeated_fingerprint_guard_is_terminal_too(self): + tool_calls = {"tool_calls": [_internal_tool_use_block()], "tool_type": "websearch"} + seen = json.dumps(tool_calls, sort_keys=True, default=str) + + result = await _run_hooks( + self.handler, + self.callback, + kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 3, "_agentic_loop_fingerprints": [seen]}, + ) + + assert self.callback.plan_calls == 0 + assert INTERNAL_TOOL_NAME not in _tool_use_names(result) + assert result["stop_reason"] == "end_turn" + + @pytest.mark.asyncio + async def test_client_declared_tool_use_is_left_alone(self): + response = _response_asking_for_another_search() + client_tool_use = {"id": "toolu_client_1", "type": "tool_use", "name": "get_weather", "input": {}} + response["content"].append(client_tool_use) + + result = await _run_hooks( + self.handler, + self.callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + response=response, + ) + + assert _tool_use_names(result) == ["get_weather"] + assert result["stop_reason"] == "tool_use" + + def test_only_the_refused_tool_calls_are_dropped(self): + """ + A block is matched on the id the rail refused, not on the tool name, so a + second block sharing that name survives when the rail never listed it. A + callback that picks its tool calls out by name hands both over and both + go, which is its own call to make; this is about not widening it here. + """ + response = _response_asking_for_another_search() + response["content"].append( + {"id": "toolu_client_1", "type": "tool_use", "name": INTERNAL_TOOL_NAME, "input": {}} + ) + + result = BaseLLMHTTPHandler._finalize_refused_agentic_response( + response=response, + tool_calls={"tool_calls": [_internal_tool_use_block()]}, + ) + + assert [block["id"] for block in result["content"] if block.get("type") == "tool_use"] == ["toolu_client_1"] + assert result["stop_reason"] == "tool_use" + + def test_tool_calls_without_ids_still_match_by_name(self): + """ + Not every callback shape carries ids on its tool calls, so the name is + still what decides when the rail refused a call that has no id. + """ + result = BaseLLMHTTPHandler._finalize_refused_agentic_response( + response=_response_asking_for_another_search(), + tool_calls={"tool_calls": [{"name": INTERNAL_TOOL_NAME, "input": {}}]}, + ) + + assert _tool_use_names(result) == [] + assert result["stop_reason"] == "end_turn" + + @pytest.mark.asyncio + async def test_streaming_caller_is_left_to_its_existing_behavior(self): + """ + A streaming caller has already sent the original message to the client, so + a finalized turn would land as a second message rather than replace the + first. The rail keeps raising there and the caller handles it as before. + """ + with pytest.raises(AgenticLoopSafetyError): + await _run_hooks( + self.handler, + self.callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + stream=True, + ) + + assert self.callback.plan_calls == 0 + + @pytest.mark.asyncio + async def test_responses_surface_is_left_to_its_existing_behavior(self): + """ + The responses surface carries a pydantic model rather than the anthropic + dict this finalizer rewrites, so it keeps raising instead of being handed + a response that was never actually finalized. + """ + with pytest.raises(AgenticLoopSafetyError): + await _run_hooks( + self.handler, + self.callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + api_surface="responses", + ) + + @pytest.mark.asyncio + async def test_non_dict_response_is_returned_untouched(self): + response = MagicMock() + + result = await _run_hooks( + self.handler, + self.callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + response=response, + ) + + assert result is response + + @pytest.mark.asyncio + async def test_converted_stream_gets_a_terminal_fake_stream(self): + """ + A converted stream is wrapped back into an Anthropic SSE stream here, the + same as every other return in this function, so a streaming client gets a + terminal stream rather than a bare dict. The interceptor turns the client's + stream into a non-streaming upstream call, so stream is False on this path + and the converted flag on the logging object is what marks it. + """ + result = await _run_hooks( + self.handler, + self.callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + converted_stream=True, + ) + + assert isinstance(result, FakeAnthropicMessagesStreamIterator) + assert result.response["stop_reason"] == "end_turn" + assert INTERNAL_TOOL_NAME not in _tool_use_names(result.response) + + def test_rails_cannot_trip_in_the_outermost_frame(self): + """ + Backs the invariant the test above relies on: at depth 0 the fingerprint set + is empty and max_loops is clamped to at least 1, so neither rail can refuse. + """ + depth, max_loops, fingerprints = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={}) + + assert depth == 0 + assert fingerprints == [] + assert max_loops >= 1 + + depth, max_loops, fingerprints = BaseLLMHTTPHandler._get_agentic_loop_settings( + kwargs={"max_agentic_loops": 0} + ) + + assert max_loops >= 1 + assert BaseLLMHTTPHandler._check_agentic_loop_safety( + tool_calls={"tool_calls": [_internal_tool_use_block()]}, + fingerprints=fingerprints, + depth=depth, + max_loops=max_loops, + model="claude-sonnet-4-5", + ) + + def test_safety_error_is_still_a_value_error(self): + assert issubclass(AgenticLoopSafetyError, ValueError) + + def test_safety_error_type_names_the_rail(self): + with pytest.raises(AgenticLoopSafetyError, match="max_agentic_loops"): + BaseLLMHTTPHandler._check_agentic_loop_safety( + tool_calls={"tool_calls": [_internal_tool_use_block()]}, + fingerprints=[], + depth=3, + max_loops=3, + model="claude-sonnet-4-5", + ) + + +class TestOuterFramePostHookStillRuns: + """ + The cap used to raise through the parent frame's await, which skipped the + parent's post-loop hook. The parent now gets its terminal response back and + finishes normally, so the blocks it was going to inject still land. + """ + + @pytest.mark.asyncio + async def test_parent_frame_injects_its_blocks_after_the_cap_trips(self, monkeypatch): + handler = BaseLLMHTTPHandler() + callback = _InterceptingCallback() + + async def fake_acreate(**call_kwargs): + return await handler._call_agentic_completion_hooks( + response=_response_asking_for_another_search(block_id="toolu_internal_2"), + model=call_kwargs["model"], + messages=call_kwargs["messages"], + anthropic_messages_provider_config=MagicMock(), + anthropic_messages_optional_request_params={}, + logging_obj=_logging_obj(callback), + stream=False, + custom_llm_provider="anthropic", + kwargs={ + key: call_kwargs[key] + for key in ("_agentic_loop_depth", "max_agentic_loops", "_agentic_loop_fingerprints") + if key in call_kwargs + }, + ) + + monkeypatch.setattr("litellm.anthropic_interface.messages.acreate", fake_acreate) + + result = await _run_hooks( + handler, + callback, + kwargs={"_agentic_loop_depth": 0, "max_agentic_loops": 1}, + ) + + assert callback.plan_calls == 1 + assert callback.post_hook_calls == 1 + assert _block_types(result)[:2] == ["server_tool_use", "web_search_tool_result"] + assert INTERNAL_TOOL_NAME not in _tool_use_names(result) + assert result["stop_reason"] == "end_turn" + + +class TestMaxAgenticLoopsConfigKnob: + def test_from_config_yaml_reads_the_knob(self): + logger = WebSearchInterceptionLogger.from_config_yaml( + {"enabled_providers": ["bedrock"], "max_agentic_loops": 7} + ) + + assert logger.max_agentic_loops == 7 + + def test_from_config_yaml_leaves_it_unset_by_default(self): + logger = WebSearchInterceptionLogger.from_config_yaml({"enabled_providers": ["bedrock"]}) + + assert logger.max_agentic_loops is None + + @pytest.mark.parametrize("bad_value", [0, -1]) + def test_out_of_range_ceilings_are_rejected_at_config_load(self, bad_value): + with pytest.raises(ValueError, match="max_agentic_loops"): + WebSearchInterceptionLogger.from_config_yaml( + {"enabled_providers": ["bedrock"], "max_agentic_loops": bad_value} + ) + + @pytest.mark.parametrize("bad_value", ["5", True, 2.5]) + def test_non_integer_ceilings_are_rejected_at_config_load(self, bad_value): + with pytest.raises(TypeError, match="max_agentic_loops"): + WebSearchInterceptionLogger.from_config_yaml( + {"enabled_providers": ["bedrock"], "max_agentic_loops": bad_value} + ) + + @pytest.mark.asyncio + async def test_knob_reaches_the_loop_settings(self): + logger = WebSearchInterceptionLogger.from_config_yaml( + {"enabled_providers": ["bedrock"], "max_agentic_loops": 7} + ) + kwargs = { + "tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}], + "litellm_params": {"custom_llm_provider": "bedrock"}, + } + + updated = await logger.async_pre_request_hook(model="claude-sonnet-4-5", messages=[], kwargs=kwargs) + + _, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs=updated) + assert max_loops == 7 + + @pytest.mark.asyncio + async def test_deployment_setting_wins_over_the_feature_setting(self): + logger = WebSearchInterceptionLogger.from_config_yaml( + {"enabled_providers": ["bedrock"], "max_agentic_loops": 7} + ) + kwargs = { + "tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}], + "litellm_params": {"custom_llm_provider": "bedrock"}, + "max_agentic_loops": 2, + } + + updated = await logger.async_pre_request_hook(model="claude-sonnet-4-5", messages=[], kwargs=kwargs) + + _, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs=updated) + assert max_loops == 2 + + @pytest.mark.asyncio + async def test_default_ceiling_applies_when_the_knob_is_unset(self): + logger = WebSearchInterceptionLogger.from_config_yaml({"enabled_providers": ["bedrock"]}) + kwargs = { + "tools": [{"type": "web_search_20250305", "name": "web_search", "max_uses": 5}], + "litellm_params": {"custom_llm_provider": "bedrock"}, + } + + updated = await logger.async_pre_request_hook(model="claude-sonnet-4-5", messages=[], kwargs=kwargs) + + assert "max_agentic_loops" not in updated + _, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs=updated) + assert max_loops == 3 From 7c02c089f650ba1de1a1e6165149d491ff57a3d1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Fri, 21 Aug 2026 19:49:39 -0700 Subject: [PATCH 2/8] docs: scope the loop-ceiling docs to the paths the fix actually covers ARCHITECTURE.md promised the clean end_turn for every intercepted request. A request that streams all the way through and one on /v1/responses both still hand back the internal tool call, so say that plainly instead. Also note that where the refused call was the only block left, the turn can come back with no text in it. On AgenticLoopSafetyError, note that the chat completions loop still raises a plain ValueError from its own copy of the rails, so nobody writes an except for this type expecting it to cover that surface too. --- .../websearch_interception/ARCHITECTURE.md | 16 ++++++++++++---- litellm/types/integrations/custom_logger.py | 5 +++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/litellm/integrations/websearch_interception/ARCHITECTURE.md b/litellm/integrations/websearch_interception/ARCHITECTURE.md index 691bb26880e..62863bd052a 100644 --- a/litellm/integrations/websearch_interception/ARCHITECTURE.md +++ b/litellm/integrations/websearch_interception/ARCHITECTURE.md @@ -235,10 +235,18 @@ model_list: Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that carries it is ignored and one request can never drive an unbounded number of upstream model calls. -When the ceiling is reached, the turn ends there and the client gets the last response back with the internal -`litellm_web_search` tool call removed and `stop_reason: end_turn`. The client never declared that tool, so -leaving the block in would hand it a tool call it has no way to answer. The answer can be less complete than -it would have been with more loops, which is the tradeoff the ceiling buys +When the ceiling is reached on a non-streaming `/v1/messages` request, the turn ends there and the client gets +the last response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`. +A streaming request the interceptor converted to non-streaming counts as one of these, since the client is +still waiting on a single response. The client never declared that tool, so leaving the block in would hand it +a tool call it has no way to answer. The answer can be less complete than it would have been with more loops, +which is the tradeoff the ceiling buys, and where the refused call was the only block left the turn can come +back with no text in it at all. + +Two paths do not get that treatment yet. A request that streams all the way through, meaning one the +interceptor did not convert, has already put its message on the wire before the ceiling is checked. And +`/v1/responses` returns its own shape that the finalizer does not rewrite, so it still hands back the internal +call. Both are tracked separately --- diff --git a/litellm/types/integrations/custom_logger.py b/litellm/types/integrations/custom_logger.py index 6b1bb2f449f..2cca16351af 100644 --- a/litellm/types/integrations/custom_logger.py +++ b/litellm/types/integrations/custom_logger.py @@ -30,6 +30,11 @@ class AgenticLoopSafetyError(ValueError): Covers both rails: the bounded-loop cap (``max_agentic_loops``) and the repeated tool-call fingerprint cycle break. Subclasses ``ValueError`` so callers that already catch the broader type keep working. + + Only the anthropic messages loop raises this today. The chat completions + loop in ``litellm_core_utils/chat_completion_agentic_loop.py`` still raises + a plain ``ValueError`` from its own copy of the same rails, so catching + this type alone will not cover that surface until it is moved over. """ From 0d1e2a5b111b769734ffac1554f6b64437e212f9 Mon Sep 17 00:00:00 2001 From: Mateo Wang Date: Fri, 21 Aug 2026 20:08:05 -0700 Subject: [PATCH 3/8] docs: correct which surfaces the loop ceiling covers --- .../websearch_interception/ARCHITECTURE.md | 20 ++++++++++--------- litellm/llms/custom_httpx/llm_http_handler.py | 16 +++++++++------ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/litellm/integrations/websearch_interception/ARCHITECTURE.md b/litellm/integrations/websearch_interception/ARCHITECTURE.md index 62863bd052a..ff49b43fa2d 100644 --- a/litellm/integrations/websearch_interception/ARCHITECTURE.md +++ b/litellm/integrations/websearch_interception/ARCHITECTURE.md @@ -235,18 +235,20 @@ model_list: Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that carries it is ignored and one request can never drive an unbounded number of upstream model calls. -When the ceiling is reached on a non-streaming `/v1/messages` request, the turn ends there and the client gets -the last response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`. -A streaming request the interceptor converted to non-streaming counts as one of these, since the client is -still waiting on a single response. The client never declared that tool, so leaving the block in would hand it -a tool call it has no way to answer. The answer can be less complete than it would have been with more loops, +When the ceiling is reached on a `/v1/messages` request, the turn ends there and the client gets the last +response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`. The client +never declared that tool, so leaving the block in would hand it a tool call it has no way to answer. The answer can be less complete than it would have been with more loops, which is the tradeoff the ceiling buys, and where the refused call was the only block left the turn can come back with no text in it at all. -Two paths do not get that treatment yet. A request that streams all the way through, meaning one the -interceptor did not convert, has already put its message on the wire before the ceiling is checked. And -`/v1/responses` returns its own shape that the finalizer does not rewrite, so it still hands back the internal -call. Both are tracked separately +Streaming is covered by the same path rather than a separate one, because interception always converts an +intercepted `stream=True` request to non-streaming before the loop runs, then rebuilds the SSE stream from the +finalized turn. So the ceiling is reached on a response the client has not seen yet either way. + +Two other surfaces do not get that treatment yet. `/v1/responses` returns its own shape that the finalizer does +not rewrite, so it still hands back the internal call. And `/v1/chat/completions` runs its own copy of these +rails in `litellm_core_utils/chat_completion_agentic_loop.py`, which still raises rather than ending the turn. +Both are tracked separately --- diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a14a89613c6..0aae700dc04 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -5186,12 +5186,16 @@ class BaseLLMHTTPHandler: """ Whether a refused rerun can still be answered with a finalized turn. - Only the non-streaming anthropic messages path can. A streaming caller - has already sent the original message to the client, so a finalized one - would arrive as a second message rather than as a replacement, and the - responses surface carries a pydantic model that the finalizer does not - rewrite. Both keep raising, which is what every surface did before this - path learned to end the turn. + Only the anthropic messages surface can. The responses surface carries a + pydantic model the finalizer does not rewrite, so it keeps raising, which + is what every surface did before this path learned to end the turn. + + Every call site passes ``stream=False`` today, because interception + converts an intercepted stream to non-streaming before the loop runs and + rebuilds the SSE stream from the finalized turn afterwards. The flag is + still checked so a streaming call site added later cannot replace a turn + already on the wire, which would reach the client as a second message + rather than as a replacement. """ return not stream and api_surface == "anthropic_messages" From 6760379b4a736a720fd2d8b0928bfc3382c57b5d Mon Sep 17 00:00:00 2001 From: Mateo Wang Date: Fri, 21 Aug 2026 20:12:54 -0700 Subject: [PATCH 4/8] test: pin the capped turn that carries only the refused call --- .../test_websearch_agentic_loop_cap.py | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py index 1d36ca76832..de3fba51eec 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py @@ -213,6 +213,40 @@ class TestCappedLoopReturnsTerminalResponse: assert _block_types(result) == ["server_tool_use", "web_search_tool_result", "text"] + @pytest.mark.asyncio + async def test_turn_carrying_only_the_refused_call_still_ends_cleanly(self): + """ + The refused call can be every block the model produced, which leaves the + turn with no content once it is dropped. That still has to come back as a + finished turn rather than as the leaked call, so the client stops instead + of waiting on a tool it cannot run, and the rest of the message survives + so the request is still billed and traceable. + + An empty turn renders as nothing, which is the ceiling being set too low + for the question rather than a malformed response. + """ + nothing_but_the_refused_call = { + "id": "msg_123", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [_internal_tool_use_block()], + "stop_reason": "tool_use", + "usage": {"input_tokens": 10, "output_tokens": 5}, + } + + result = await _run_hooks( + self.handler, + self.callback, + kwargs={"_agentic_loop_depth": 3, "max_agentic_loops": 3}, + response=nothing_but_the_refused_call, + ) + + assert result["content"] == [] + assert result["stop_reason"] == "end_turn" + assert result["usage"] == {"input_tokens": 10, "output_tokens": 5} + assert result["id"] == "msg_123" + @pytest.mark.asyncio async def test_no_follow_up_model_call_is_planned(self): """ From 206e3b8560728042fc623af65d8f5174e91b61e1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 08:51:30 -0700 Subject: [PATCH 5/8] docs: say the loop ceiling covers non-streaming /v1/messages --- .../websearch_interception/ARCHITECTURE.md | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/websearch_interception/ARCHITECTURE.md b/litellm/integrations/websearch_interception/ARCHITECTURE.md index ff49b43fa2d..0cce648003e 100644 --- a/litellm/integrations/websearch_interception/ARCHITECTURE.md +++ b/litellm/integrations/websearch_interception/ARCHITECTURE.md @@ -235,15 +235,17 @@ model_list: Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that carries it is ignored and one request can never drive an unbounded number of upstream model calls. -When the ceiling is reached on a `/v1/messages` request, the turn ends there and the client gets the last -response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`. The client -never declared that tool, so leaving the block in would hand it a tool call it has no way to answer. The answer can be less complete than it would have been with more loops, -which is the tradeoff the ceiling buys, and where the refused call was the only block left the turn can come -back with no text in it at all. +When the ceiling is reached on a non-streaming `/v1/messages` request, the turn ends there and the client gets +the last response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`. +The client never declared that tool, so leaving the block in would hand it a tool call it has no way to answer. +The answer can be less complete than it would have been with more loops, which is the tradeoff the ceiling +buys. Where the refused call was the only block left, the turn comes back with no text in it at all. -Streaming is covered by the same path rather than a separate one, because interception always converts an -intercepted `stream=True` request to non-streaming before the loop runs, then rebuilds the SSE stream from the -finalized turn. So the ceiling is reached on a response the client has not seen yet either way. +Non-streaming is not a limitation on the client here, because a client that asked for a stream gets the same +treatment. Interception converts an intercepted `stream=True` request to non-streaming before the loop runs and +rebuilds the SSE stream from the finalized turn afterwards, so the ceiling is always reached on a response the +client has not seen yet. The guard is written against the flag anyway, so a caller added later that reaches the +loop with a stream already open keeps raising rather than replacing a turn that is halfway to the client. Two other surfaces do not get that treatment yet. `/v1/responses` returns its own shape that the finalizer does not rewrite, so it still hands back the internal call. And `/v1/chat/completions` runs its own copy of these From 0485b3fcd42ab704f7cca0e7627b79766f62f9ed Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 10:31:56 -0700 Subject: [PATCH 6/8] fix: emit content_block_start for every block in the rebuilt stream A capped turn on a streaming request is rebuilt into SSE by FakeAnthropicMessagesStreamIterator. It emitted content_block_stop for every block but content_block_start only for text, thinking, redacted_thinking and tool_use, so a web search turn's server_tool_use and web_search_tool_result blocks produced stops with no matching start. Anthropic's SDK accumulator appends on content_block_start and then indexes content[event.index] on content_block_delta, so the orphan stops shifted every later index and client.messages.stream() raised IndexError on the text block. Unknown block types now pass through with a start of their own, which keeps position equal to index. Also corrects two claims that said no current caller reaches the loop with stream=True. AgenticStreamingIterator does, and it keeps raising, because its events are already on the wire. --- .../websearch_interception/ARCHITECTURE.md | 5 +- .../messages/fake_stream_iterator.py | 8 ++ litellm/llms/custom_httpx/llm_http_handler.py | 13 +-- .../test_websearch_agentic_loop_cap.py | 93 +++++++++++++++++++ 4 files changed, 111 insertions(+), 8 deletions(-) diff --git a/litellm/integrations/websearch_interception/ARCHITECTURE.md b/litellm/integrations/websearch_interception/ARCHITECTURE.md index 0cce648003e..b1485b9b680 100644 --- a/litellm/integrations/websearch_interception/ARCHITECTURE.md +++ b/litellm/integrations/websearch_interception/ARCHITECTURE.md @@ -244,8 +244,9 @@ buys. Where the refused call was the only block left, the turn comes back with n Non-streaming is not a limitation on the client here, because a client that asked for a stream gets the same treatment. Interception converts an intercepted `stream=True` request to non-streaming before the loop runs and rebuilds the SSE stream from the finalized turn afterwards, so the ceiling is always reached on a response the -client has not seen yet. The guard is written against the flag anyway, so a caller added later that reaches the -loop with a stream already open keeps raising rather than replacing a turn that is halfway to the client. +client has not seen yet. `AgenticStreamingIterator` is the one caller that reaches the loop with its events +already on the wire, and it keeps raising, because a finalized turn would arrive there as a second message +rather than as a replacement. Two other surfaces do not get that treatment yet. `/v1/responses` returns its own shape that the finalizer does not rewrite, so it still hands back the internal call. And `/v1/chat/completions` runs its own copy of these diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py b/litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py index 215d4a5b42b..14f1b7697cf 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/fake_stream_iterator.py @@ -113,6 +113,14 @@ class FakeAnthropicMessagesStreamIterator: } chunks.append(f"event: content_block_delta\ndata: {json.dumps(content_block_delta)}\n\n".encode()) + else: + passthrough_start: Final = { + "type": "content_block_start", + "index": index, + "content_block": block_dict, + } + chunks.append(f"event: content_block_start\ndata: {json.dumps(passthrough_start)}\n\n".encode()) + content_block_stop: Final = {"type": "content_block_stop", "index": index} chunks.append(f"event: content_block_stop\ndata: {json.dumps(content_block_stop)}\n\n".encode()) return chunks diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 0aae700dc04..ebc76e6c7ea 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -5190,12 +5190,13 @@ class BaseLLMHTTPHandler: pydantic model the finalizer does not rewrite, so it keeps raising, which is what every surface did before this path learned to end the turn. - Every call site passes ``stream=False`` today, because interception - converts an intercepted stream to non-streaming before the loop runs and - rebuilds the SSE stream from the finalized turn afterwards. The flag is - still checked so a streaming call site added later cannot replace a turn - already on the wire, which would reach the client as a second message - rather than as a replacement. + The messages and responses call sites pass ``stream=False``, because + interception converts an intercepted stream to non-streaming before the + loop runs and rebuilds the SSE stream from the finalized turn + afterwards. ``AgenticStreamingIterator`` passes ``stream=True``, and + that path keeps raising: its events are already on the wire, so a + finalized turn would reach the client as a second message rather than + as a replacement. """ return not stream and api_surface == "anthropic_messages" diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py index de3fba51eec..91148f7cf6d 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py @@ -559,3 +559,96 @@ class TestMaxAgenticLoopsConfigKnob: assert "max_agentic_loops" not in updated _, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs=updated) assert max_loops == 3 + + +def _stream_events(response: dict) -> list[dict]: + events: list[dict] = [] + for chunk in FakeAnthropicMessagesStreamIterator(response=response): + for line in chunk.decode().splitlines(): + if line.startswith("data: "): + events.append(json.loads(line[len("data: ") :])) + return events + + +class TestRebuiltStreamIsWellFormed: + """ + A capped turn is rebuilt into SSE by FakeAnthropicMessagesStreamIterator. + + Anthropic's SDK accumulator appends on content_block_start and then indexes + content[event.index] on content_block_delta, so a block that stops without + ever starting shifts every later index and the accumulator raises + IndexError. A web search turn carries server_tool_use and + web_search_tool_result blocks, which is exactly where that used to happen. + """ + + @staticmethod + def _capped_search_turn() -> dict: + return { + "id": "msg_01", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "stop_reason": "end_turn", + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_01", + "name": "web_search", + "input": {"query": "on-demand H100 hourly price"}, + }, + { + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_01", + "content": [ + { + "type": "web_search_result", + "url": "https://example.com/h100", + "title": "H100 pricing", + } + ], + }, + {"type": "text", "text": "AWS lists the H100 at $12.29 an hour."}, + ], + "usage": {"input_tokens": 100, "output_tokens": 20}, + } + + def test_every_content_block_stop_has_a_matching_start(self): + events = _stream_events(self._capped_search_turn()) + + started = [event["index"] for event in events if event["type"] == "content_block_start"] + stopped = [event["index"] for event in events if event["type"] == "content_block_stop"] + + assert started == [0, 1, 2] + assert stopped == [0, 1, 2] + + def test_no_delta_indexes_past_the_blocks_started_before_it(self): + events = _stream_events(self._capped_search_turn()) + + blocks_started = 0 + for event in events: + if event["type"] == "content_block_start": + blocks_started += 1 + elif event["type"] == "content_block_delta": + assert event["index"] < blocks_started + + def test_search_blocks_reach_the_client(self): + events = _stream_events(self._capped_search_turn()) + + started_types = [ + event["content_block"]["type"] for event in events if event["type"] == "content_block_start" + ] + + assert started_types == ["server_tool_use", "web_search_tool_result", "text"] + + def test_the_search_result_survives_the_rebuild_intact(self): + events = _stream_events(self._capped_search_turn()) + + result_block = next( + event["content_block"] + for event in events + if event["type"] == "content_block_start" + and event["content_block"]["type"] == "web_search_tool_result" + ) + + assert result_block["tool_use_id"] == "srvtoolu_01" + assert result_block["content"][0]["url"] == "https://example.com/h100" From 19e077ab510240a3d0c9993e27e8d635fff6d318 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 10:53:13 -0700 Subject: [PATCH 7/8] fix: validate max_agentic_loops wherever it is set The ceiling was only checked at the feature level, on litellm_settings.websearch_interception_params. The per-deployment litellm_params.max_agentic_loops, which wins over it, went straight into int(kwargs.get("max_agentic_loops", 3) or 3), so a 0 was swallowed by the falsy fallback and read as the default 3. Asking for the tightest ceiling handed you the loosest one. A non-integer booted the proxy and then failed every request to that model with "invalid literal for int() with base 10". Both settings now share one validator, which names the field it rejected, and the per-deployment value is checked while the model list is read at startup so a bad value stops the proxy rather than surfacing per request. The check sits in load_config rather than on LiteLLM_Params because the proxy builds its router with ignore_invalid_deployments=True, where a validation error drops the deployment silently instead of refusing to start. This is the same placement the complexity_router_config plugin check already uses. Chat completions read the same key through a separate path that turned 0 into 1 and true into a ceiling of 1, so it now shares the validator too and the key means one thing on both surfaces. --- .../websearch_interception/ARCHITECTURE.md | 5 ++ .../websearch_interception/handler.py | 18 ++---- .../agentic_loop_settings.py | 35 ++++++++++++ .../chat_completion_agentic_loop.py | 9 ++- litellm/llms/custom_httpx/llm_http_handler.py | 11 +++- litellm/proxy/proxy_server.py | 25 +++++++++ .../test_websearch_agentic_loop_cap.py | 56 ++++++++++++++++++- .../proxy/proxy_server/test_proxy_config.py | 53 ++++++++++++++++++ 8 files changed, 192 insertions(+), 20 deletions(-) create mode 100644 litellm/litellm_core_utils/agentic_loop_settings.py diff --git a/litellm/integrations/websearch_interception/ARCHITECTURE.md b/litellm/integrations/websearch_interception/ARCHITECTURE.md index b1485b9b680..4ea7a7ae527 100644 --- a/litellm/integrations/websearch_interception/ARCHITECTURE.md +++ b/litellm/integrations/websearch_interception/ARCHITECTURE.md @@ -235,6 +235,11 @@ model_list: Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that carries it is ignored and one request can never drive an unbounded number of upstream model calls. +Both places are validated at config load, and a value that is not an integer of at least 1 stops the proxy +from starting rather than surfacing later. The per-deployment one is checked while the model list is read, +not on `LiteLLM_Params`, because the proxy builds its router with `ignore_invalid_deployments=True` and a +validator down there would drop the deployment silently instead of refusing to start. + When the ceiling is reached on a non-streaming `/v1/messages` request, the turn ends there and the client gets the last response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`. The client never declared that tool, so leaving the block in would hand it a tool call it has no way to answer. diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 760824f820f..13a16947fb4 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -31,6 +31,9 @@ from litellm.integrations.websearch_interception.tools import ( from litellm.integrations.websearch_interception.transformation import ( WebSearchTransformation, ) +from litellm.litellm_core_utils.agentic_loop_settings import ( + validated_max_agentic_loops, +) from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, @@ -150,21 +153,8 @@ class WebSearchInterceptionLogger(CustomLogger): def _validated_max_agentic_loops(max_agentic_loops: object) -> int | None: """ Reject loop ceilings the agentic loop cannot honor, at config load time. - - ``bool`` is excluded explicitly because it is an ``int`` subclass, so - ``max_agentic_loops: true`` would otherwise be read as a ceiling of 1. """ - if max_agentic_loops is None: - return None - if isinstance(max_agentic_loops, bool) or not isinstance(max_agentic_loops, int): - raise TypeError( - f"websearch_interception_params.max_agentic_loops must be an integer, got {max_agentic_loops!r}" - ) - if max_agentic_loops < 1: - raise ValueError( - f"websearch_interception_params.max_agentic_loops must be at least 1, got {max_agentic_loops}" - ) - return max_agentic_loops + return validated_max_agentic_loops(max_agentic_loops, field="websearch_interception_params.max_agentic_loops") async def try_short_circuit_search( self, diff --git a/litellm/litellm_core_utils/agentic_loop_settings.py b/litellm/litellm_core_utils/agentic_loop_settings.py new file mode 100644 index 00000000000..538f2d64c1b --- /dev/null +++ b/litellm/litellm_core_utils/agentic_loop_settings.py @@ -0,0 +1,35 @@ +""" +Shared validation for the agentic loop ceiling. + +``max_agentic_loops`` can be set in two places, and the two disagreed about +what a bad value means. The feature-level +``litellm_settings.websearch_interception_params.max_agentic_loops`` was +checked at config load, while a per-deployment +``model_list[].litellm_params.max_agentic_loops`` was passed straight through +to ``int(... or 3)``. That let a per-deployment ``0`` read as the default 3, +turning the tightest ceiling into the loosest one, and let a per-deployment +``"three"`` boot the proxy and then fail every request to that model. + +Both settings now go through :func:`validated_max_agentic_loops`, which names +the field it rejected so the error says which line of the config to fix. +""" + +from typing import Final + +DEFAULT_MAX_AGENTIC_LOOPS: Final = 3 + + +def validated_max_agentic_loops(max_agentic_loops: object, field: str) -> int | None: + """ + Return ``max_agentic_loops`` as an int, or raise naming ``field``. + + ``bool`` is excluded explicitly because it is an ``int`` subclass, so + ``max_agentic_loops: true`` would otherwise be read as a ceiling of 1. + """ + if max_agentic_loops is None: + return None + if isinstance(max_agentic_loops, bool) or not isinstance(max_agentic_loops, int): + raise TypeError(f"{field} must be an integer, got {max_agentic_loops!r}") + if max_agentic_loops < 1: + raise ValueError(f"{field} must be at least 1, got {max_agentic_loops}") + return max_agentic_loops diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index b91c1785a54..07bed1f88ad 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -5,6 +5,10 @@ from typing import Final, cast from litellm._logging import verbose_logger from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.agentic_loop_settings import ( + DEFAULT_MAX_AGENTIC_LOOPS, + validated_max_agentic_loops, +) from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, @@ -52,7 +56,10 @@ def _coerce_int(value: object, default: int) -> int: def _agentic_loop_settings(kwargs: dict[str, object]) -> tuple[int, int, list[str]]: depth: Final = _coerce_int(kwargs.get("_agentic_loop_depth"), 0) - max_loops: Final = max(_coerce_int(kwargs.get("max_agentic_loops"), 3), 1) + configured: Final = validated_max_agentic_loops( + kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops" + ) + max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured raw_fingerprints: Final = kwargs.get("_agentic_loop_fingerprints") fingerprints: Final = [str(fp) for fp in raw_fingerprints] if isinstance(raw_fingerprints, list) else [] return depth, max_loops, fingerprints diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ebc76e6c7ea..862d98f65e6 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -19,6 +19,10 @@ import litellm.types.utils from litellm._logging import _redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES +from litellm.litellm_core_utils.agentic_loop_settings import ( + DEFAULT_MAX_AGENTIC_LOOPS, + validated_max_agentic_loops, +) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.realtime_errors import realtime_error_event, websocket_close_reason from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming @@ -5078,9 +5082,12 @@ class BaseLLMHTTPHandler: @staticmethod def _get_agentic_loop_settings(kwargs: dict) -> tuple[int, int, list[str]]: depth: Final = int(kwargs.get("_agentic_loop_depth", 0) or 0) - max_loops: Final = int(kwargs.get("max_agentic_loops", 3) or 3) + configured: Final = validated_max_agentic_loops( + kwargs.get("max_agentic_loops"), field="litellm_params.max_agentic_loops" + ) + max_loops: Final = DEFAULT_MAX_AGENTIC_LOOPS if configured is None else configured fingerprints: Final = list(kwargs.get("_agentic_loop_fingerprints", []) or []) - return depth, max(max_loops, 1), fingerprints + return depth, max_loops, fingerprints @staticmethod def _has_agentic_completion_hook(logging_obj: LiteLLMLoggingObj) -> bool: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 9f16584340d..7dced4e26b6 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -254,6 +254,9 @@ from litellm.exceptions import RejectedRequestError from litellm.integrations.custom_guardrail import ModifyResponseException from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting +from litellm.litellm_core_utils.agentic_loop_settings import ( + validated_max_agentic_loops, +) from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.core_helpers import ( _get_parent_otel_span_from_kwargs, @@ -4081,6 +4084,27 @@ def resolve_complexity_router_plugins( complexity_router_config["classifier_plugin"] = resolved_classifier # rebind-ok: out-param, resolved in place +def validate_deployment_max_agentic_loops(model: Mapping[str, Any]) -> None: + """ + Reject a per-deployment `max_agentic_loops` the agentic loop cannot honor. + + Checked here rather than on `LiteLLM_Params` because the proxy builds its + router with `ignore_invalid_deployments=True`, so a validator down there + turns a bad value into a silently missing model instead of a refusal to + start. Left unchecked entirely, a `0` used to read as the default ceiling + of 3 and a non-integer failed every request to that model instead. + """ + litellm_params: Final = model.get("litellm_params") or {} + if "max_agentic_loops" not in litellm_params: + return + + model_name: Final = model.get("model_name", "") + validated_max_agentic_loops( + litellm_params["max_agentic_loops"], + field=f"litellm_params.max_agentic_loops on model {model_name!r}", + ) + + def pin_complexity_router_model_id(model: dict) -> None: # mutable-ok: out-param, model_info is stamped in place """ Stamps `model_info.id` from the raw litellm_params before plugin resolution swaps @@ -5416,6 +5440,7 @@ class ProxyConfig: for k, v in model["litellm_params"].items(): if isinstance(v, str) and v.startswith("os.environ/"): model["litellm_params"][k] = get_secret(v) + validate_deployment_max_agentic_loops(model) pin_complexity_router_model_id(model) complexity_router_config = model["litellm_params"].get("complexity_router_config") if isinstance(complexity_router_config, dict): diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py index 91148f7cf6d..327b1b066b4 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py @@ -24,6 +24,7 @@ from litellm.integrations.websearch_interception.handler import ( from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import ( FakeAnthropicMessagesStreamIterator, ) +from litellm.litellm_core_utils.agentic_loop_settings import DEFAULT_MAX_AGENTIC_LOOPS from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.integrations.custom_logger import ( AgenticLoopPlan, @@ -409,7 +410,7 @@ class TestCappedLoopReturnsTerminalResponse: def test_rails_cannot_trip_in_the_outermost_frame(self): """ Backs the invariant the test above relies on: at depth 0 the fingerprint set - is empty and max_loops is clamped to at least 1, so neither rail can refuse. + is empty and the ceiling is at least 1, so neither rail can refuse. """ depth, max_loops, fingerprints = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={}) @@ -418,10 +419,10 @@ class TestCappedLoopReturnsTerminalResponse: assert max_loops >= 1 depth, max_loops, fingerprints = BaseLLMHTTPHandler._get_agentic_loop_settings( - kwargs={"max_agentic_loops": 0} + kwargs={"max_agentic_loops": 1} ) - assert max_loops >= 1 + assert max_loops == 1 assert BaseLLMHTTPHandler._check_agentic_loop_safety( tool_calls={"tool_calls": [_internal_tool_use_block()]}, fingerprints=fingerprints, @@ -570,6 +571,55 @@ def _stream_events(response: dict) -> list[dict]: return events +class TestBothCeilingKnobsAreValidated: + """ + ``max_agentic_loops`` is settable per deployment and feature-wide, and the + per-deployment one wins. Only the feature-wide one used to be checked, so a + per-deployment ``0`` was swallowed by an ``or 3`` and read as the default 3, + handing the loosest ceiling to whoever asked for the tightest. + """ + + def test_a_per_deployment_zero_is_rejected_not_read_as_the_default(self): + with pytest.raises(ValueError, match="must be at least 1, got 0"): + BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 0}) + + def test_a_per_deployment_non_integer_names_the_field_it_came_from(self): + with pytest.raises(TypeError, match=r"litellm_params\.max_agentic_loops must be an integer"): + BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": "three"}) + + def test_a_per_deployment_true_is_not_read_as_a_ceiling_of_one(self): + with pytest.raises(TypeError, match="must be an integer"): + BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": True}) + + def test_an_absent_ceiling_falls_back_to_the_shared_default(self): + _, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={}) + + assert max_loops == DEFAULT_MAX_AGENTIC_LOOPS + + def test_an_explicit_none_falls_back_to_the_shared_default(self): + _, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": None}) + + assert max_loops == DEFAULT_MAX_AGENTIC_LOOPS + + def test_a_valid_per_deployment_ceiling_is_passed_through(self): + _, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 6}) + + assert max_loops == 6 + + @pytest.mark.parametrize("rejected", [0, -1, "three", True]) + def test_the_two_knobs_reject_the_same_values(self, rejected): + with pytest.raises((TypeError, ValueError)): + WebSearchInterceptionLogger(max_agentic_loops=rejected) + with pytest.raises((TypeError, ValueError)): + BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": rejected}) + + def test_each_knob_names_its_own_config_field(self): + with pytest.raises(ValueError, match=r"websearch_interception_params\.max_agentic_loops"): + WebSearchInterceptionLogger(max_agentic_loops=0) + with pytest.raises(ValueError, match=r"litellm_params\.max_agentic_loops"): + BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 0}) + + class TestRebuiltStreamIsWellFormed: """ A capped turn is rebuilt into SSE by FakeAnthropicMessagesStreamIterator. diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index b0b2c68e30d..fa8355ad8c4 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -26,6 +26,7 @@ from litellm.proxy.proxy_server import ( _scrub_guardrail_inner, resolve_complexity_router_plugins, resolve_routing_plugins, + validate_deployment_max_agentic_loops, ) from .conftest import normalize @@ -153,6 +154,58 @@ def test_resolve_complexity_router_plugins_resolves_dotted_path_to_live_instance assert type(config["plugins"][0]).__name__ == "_Plugin" +def test_validate_deployment_max_agentic_loops_allows_a_deployment_without_the_key(): + model = {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o"}} + + validate_deployment_max_agentic_loops(model) + + assert "max_agentic_loops" not in model["litellm_params"] + + +def test_validate_deployment_max_agentic_loops_leaves_a_valid_ceiling_alone(): + model = {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": 5}} + + validate_deployment_max_agentic_loops(model) + + assert model["litellm_params"]["max_agentic_loops"] == 5 + + +def test_validate_deployment_max_agentic_loops_rejects_zero(): + """ + A per-deployment 0 used to be swallowed by an `or 3` and read as the default + ceiling of 3, handing the loosest setting to whoever asked for the tightest. + """ + with pytest.raises(ValueError, match="must be at least 1, got 0"): + validate_deployment_max_agentic_loops( + {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": 0}} + ) + + +def test_validate_deployment_max_agentic_loops_rejects_a_non_integer(): + """ + A per-deployment non-integer used to let the proxy boot and then fail every + request to that model with `invalid literal for int() with base 10`. + """ + with pytest.raises(TypeError, match="must be an integer"): + validate_deployment_max_agentic_loops( + {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": "three"}} + ) + + +def test_validate_deployment_max_agentic_loops_rejects_a_bool(): + with pytest.raises(TypeError, match="must be an integer"): + validate_deployment_max_agentic_loops( + {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": True}} + ) + + +def test_validate_deployment_max_agentic_loops_names_the_offending_model(): + with pytest.raises(ValueError, match="on model 'claude-sonnet-4-5'"): + validate_deployment_max_agentic_loops( + {"model_name": "claude-sonnet-4-5", "litellm_params": {"max_agentic_loops": -1}} + ) + + def test_resolve_complexity_router_plugins_rejects_non_routing_plugin_object(tmp_path): plugin_file = tmp_path / "bad_plugin.py" plugin_file.write_text("not_a_plugin = object()\n") From b103edb588cecaf9a90b2c4768a431055ec8c38f Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 22 Aug 2026 11:24:22 -0700 Subject: [PATCH 8/8] fix: keep accepting a loop ceiling that spells a whole number The ceiling used to go through `int(... or 3)`, so anything `int()` accepted worked. Tightening the new shared validator to `isinstance(int)` turned a config that boots today into a proxy that refuses to start, because `max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS` is resolved to a string before it reaches either check, and a YAML-quoted "5" is a string too. Accept ints, integral floats, and strings that parse to a whole number. Keep refusing bools, fractional floats, words, and anything below 1. --- .../agentic_loop_settings.py | 36 ++++++++++--- .../test_websearch_agentic_loop_cap.py | 52 ++++++++++++++++++- .../proxy/proxy_server/test_proxy_config.py | 13 +++++ 3 files changed, 94 insertions(+), 7 deletions(-) diff --git a/litellm/litellm_core_utils/agentic_loop_settings.py b/litellm/litellm_core_utils/agentic_loop_settings.py index 538f2d64c1b..3dd8d437aef 100644 --- a/litellm/litellm_core_utils/agentic_loop_settings.py +++ b/litellm/litellm_core_utils/agentic_loop_settings.py @@ -12,6 +12,11 @@ turning the tightest ceiling into the loosest one, and let a per-deployment Both settings now go through :func:`validated_max_agentic_loops`, which names the field it rejected so the error says which line of the config to fix. + +Anything that spells a whole number is still accepted, because the old +``int(... or 3)`` accepted those and a ceiling is routinely parameterized as +``max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS``, which resolves to a +string. Rejecting ``"5"`` would stop such a proxy from booting on upgrade. """ from typing import Final @@ -19,17 +24,36 @@ from typing import Final DEFAULT_MAX_AGENTIC_LOOPS: Final = 3 -def validated_max_agentic_loops(max_agentic_loops: object, field: str) -> int | None: +def _as_whole_number(value: object) -> int | None: """ - Return ``max_agentic_loops`` as an int, or raise naming ``field``. + Return ``value`` as an int when it spells a whole number, else ``None``. ``bool`` is excluded explicitly because it is an ``int`` subclass, so ``max_agentic_loops: true`` would otherwise be read as a ceiling of 1. """ + if isinstance(value, bool): + return None + if isinstance(value, int): + return value + if isinstance(value, float): + return int(value) if value.is_integer() else None + if isinstance(value, str): + try: + return int(value.strip()) + except ValueError: + return None + return None + + +def validated_max_agentic_loops(max_agentic_loops: object, field: str) -> int | None: + """ + Return ``max_agentic_loops`` as an int, or raise naming ``field``. + """ if max_agentic_loops is None: return None - if isinstance(max_agentic_loops, bool) or not isinstance(max_agentic_loops, int): + ceiling: Final = _as_whole_number(max_agentic_loops) + if ceiling is None: raise TypeError(f"{field} must be an integer, got {max_agentic_loops!r}") - if max_agentic_loops < 1: - raise ValueError(f"{field} must be at least 1, got {max_agentic_loops}") - return max_agentic_loops + if ceiling < 1: + raise ValueError(f"{field} must be at least 1, got {ceiling}") + return ceiling diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py index 327b1b066b4..40fd8c4e9e6 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_agentic_loop_cap.py @@ -26,6 +26,7 @@ from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_itera ) from litellm.litellm_core_utils.agentic_loop_settings import DEFAULT_MAX_AGENTIC_LOOPS from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.secret_managers.main import get_secret from litellm.types.integrations.custom_logger import ( AgenticLoopPlan, AgenticLoopRequestPatch, @@ -509,13 +510,25 @@ class TestMaxAgenticLoopsConfigKnob: {"enabled_providers": ["bedrock"], "max_agentic_loops": bad_value} ) - @pytest.mark.parametrize("bad_value", ["5", True, 2.5]) + @pytest.mark.parametrize("bad_value", ["three", True, 2.5]) def test_non_integer_ceilings_are_rejected_at_config_load(self, bad_value): with pytest.raises(TypeError, match="max_agentic_loops"): WebSearchInterceptionLogger.from_config_yaml( {"enabled_providers": ["bedrock"], "max_agentic_loops": bad_value} ) + def test_a_ceiling_spelled_as_a_string_is_read_at_config_load(self): + """ + `max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS` resolves to a string + before it reaches the knob, so refusing "5" would break a config that + works today. + """ + logger = WebSearchInterceptionLogger.from_config_yaml( + {"enabled_providers": ["bedrock"], "max_agentic_loops": "5"} + ) + + assert logger.max_agentic_loops == 5 + @pytest.mark.asyncio async def test_knob_reaches_the_loop_settings(self): logger = WebSearchInterceptionLogger.from_config_yaml( @@ -620,6 +633,43 @@ class TestBothCeilingKnobsAreValidated: BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 0}) +class TestACeilingThatSpellsAWholeNumberStillWorks: + """ + The ceiling used to go through ``int(... or 3)``, which accepted anything + ``int()`` accepted. A ceiling is routinely parameterized as + ``max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS``, and ``get_secret`` + hands that back as the string ``"5"``, so tightening the check to + ``isinstance(int)`` would stop such a proxy from booting on upgrade. + """ + + @pytest.mark.parametrize("spelled", ["5", " 5 ", 5.0]) + def test_a_ceiling_that_spells_five_is_accepted_by_both_knobs(self, spelled): + _, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": spelled}) + + assert max_loops == 5 + assert WebSearchInterceptionLogger(max_agentic_loops=spelled).max_agentic_loops == 5 + + def test_an_env_var_sourced_ceiling_survives_secret_resolution(self, monkeypatch): + monkeypatch.setenv("MAX_AGENTIC_LOOPS_UNDER_TEST", "7") + resolved = get_secret("os.environ/MAX_AGENTIC_LOOPS_UNDER_TEST") + + assert isinstance(resolved, str) + _, max_loops, _ = BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": resolved}) + assert max_loops == 7 + + def test_a_spelled_zero_is_still_refused_and_reports_the_number(self): + with pytest.raises(ValueError, match="must be at least 1, got 0"): + BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": "0"}) + + def test_a_word_is_still_refused(self): + with pytest.raises(TypeError, match=r"litellm_params\.max_agentic_loops must be an integer"): + BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": "three"}) + + def test_a_fractional_ceiling_is_refused_rather_than_truncated(self): + with pytest.raises(TypeError, match="must be an integer"): + BaseLLMHTTPHandler._get_agentic_loop_settings(kwargs={"max_agentic_loops": 5.5}) + + class TestRebuiltStreamIsWellFormed: """ A capped turn is rebuilt into SSE by FakeAnthropicMessagesStreamIterator. diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index fa8355ad8c4..ee0de8840f6 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -199,6 +199,19 @@ def test_validate_deployment_max_agentic_loops_rejects_a_bool(): ) +def test_validate_deployment_max_agentic_loops_accepts_a_ceiling_from_an_env_var(): + """ + `max_agentic_loops: os.environ/MAX_AGENTIC_LOOPS` is resolved to a string + before this check runs, and the old `int(... or 3)` accepted that, so + refusing it here would stop an already working proxy from booting. + """ + model = {"model_name": "gpt-4o", "litellm_params": {"model": "gpt-4o", "max_agentic_loops": "5"}} + + validate_deployment_max_agentic_loops(model) + + assert model["litellm_params"]["max_agentic_loops"] == "5" + + def test_validate_deployment_max_agentic_loops_names_the_offending_model(): with pytest.raises(ValueError, match="on model 'claude-sonnet-4-5'"): validate_deployment_max_agentic_loops(