From 0d1e2a5b111b769734ffac1554f6b64437e212f9 Mon Sep 17 00:00:00 2001 From: Mateo Wang Date: Fri, 21 Aug 2026 20:08:05 -0700 Subject: [PATCH] docs: correct which surfaces the loop ceiling covers --- .../websearch_interception/ARCHITECTURE.md | 20 ++++++++++--------- litellm/llms/custom_httpx/llm_http_handler.py | 16 +++++++++------ 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/litellm/integrations/websearch_interception/ARCHITECTURE.md b/litellm/integrations/websearch_interception/ARCHITECTURE.md index 62863bd052a..ff49b43fa2d 100644 --- a/litellm/integrations/websearch_interception/ARCHITECTURE.md +++ b/litellm/integrations/websearch_interception/ARCHITECTURE.md @@ -235,18 +235,20 @@ model_list: Clients cannot set it. `max_agentic_loops` is on the proxy's untrusted-field list, so a request body that carries it is ignored and one request can never drive an unbounded number of upstream model calls. -When the ceiling is reached on a non-streaming `/v1/messages` request, the turn ends there and the client gets -the last response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`. -A streaming request the interceptor converted to non-streaming counts as one of these, since the client is -still waiting on a single response. The client never declared that tool, so leaving the block in would hand it -a tool call it has no way to answer. The answer can be less complete than it would have been with more loops, +When the ceiling is reached on a `/v1/messages` request, the turn ends there and the client gets the last +response back with the internal `litellm_web_search` tool call removed and `stop_reason: end_turn`. The client +never declared that tool, so leaving the block in would hand it a tool call it has no way to answer. The answer can be less complete than it would have been with more loops, which is the tradeoff the ceiling buys, and where the refused call was the only block left the turn can come back with no text in it at all. -Two paths do not get that treatment yet. A request that streams all the way through, meaning one the -interceptor did not convert, has already put its message on the wire before the ceiling is checked. And -`/v1/responses` returns its own shape that the finalizer does not rewrite, so it still hands back the internal -call. Both are tracked separately +Streaming is covered by the same path rather than a separate one, because interception always converts an +intercepted `stream=True` request to non-streaming before the loop runs, then rebuilds the SSE stream from the +finalized turn. So the ceiling is reached on a response the client has not seen yet either way. + +Two other surfaces do not get that treatment yet. `/v1/responses` returns its own shape that the finalizer does +not rewrite, so it still hands back the internal call. And `/v1/chat/completions` runs its own copy of these +rails in `litellm_core_utils/chat_completion_agentic_loop.py`, which still raises rather than ending the turn. +Both are tracked separately --- diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index a14a89613c6..0aae700dc04 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -5186,12 +5186,16 @@ class BaseLLMHTTPHandler: """ Whether a refused rerun can still be answered with a finalized turn. - Only the non-streaming anthropic messages path can. A streaming caller - has already sent the original message to the client, so a finalized one - would arrive as a second message rather than as a replacement, and the - responses surface carries a pydantic model that the finalizer does not - rewrite. Both keep raising, which is what every surface did before this - path learned to end the turn. + Only the anthropic messages surface can. The responses surface carries a + pydantic model the finalizer does not rewrite, so it keeps raising, which + is what every surface did before this path learned to end the turn. + + Every call site passes ``stream=False`` today, because interception + converts an intercepted stream to non-streaming before the loop runs and + rebuilds the SSE stream from the finalized turn afterwards. The flag is + still checked so a streaming call site added later cannot replace a turn + already on the wire, which would reach the client as a second message + rather than as a replacement. """ return not stream and api_surface == "anthropic_messages"