fix: guard short-circuit against providers with native agentic loop

- Skip short-circuit for providers that have a BaseAnthropicMessagesConfig
  (bedrock, vertex_ai, azure_ai, anthropic) — they use the agentic loop
  which includes a follow-up LLM synthesis step. Short-circuiting would
  return raw search text instead of an LLM-synthesized answer.
- Add fallback to litellm.get_llm_provider() for custom_llm_provider
  derivation when litellm_params is overwritten by kwargs.
- Add test for bedrock guard.

Addresses Greptile review comments #3 and #4.
This commit is contained in:
Jonathan Barazany 2026-03-20 01:07:20 +02:00
parent 141ad04955
commit 32cb6f0cd9
3 changed files with 54 additions and 1 deletions

View file

@ -29,6 +29,7 @@ from litellm.types.integrations.websearch_interception import (
WebSearchInterceptionConfig,
)
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
class WebSearchInterceptionLogger(CustomLogger):
@ -106,6 +107,28 @@ class WebSearchInterceptionLogger(CustomLogger):
):
return None
# Only short-circuit for providers without native Anthropic Messages
# support. Providers that have a BaseAnthropicMessagesConfig (bedrock,
# vertex_ai, azure_ai, anthropic) already use the agentic loop, which
# includes a follow-up LLM call to synthesize the answer from search
# results. Short-circuiting those would skip that synthesis step and
# return raw search text — a regression for existing users.
try:
provider_enum = LlmProviders(provider_str)
anthropic_config = (
ProviderConfigManager.get_provider_anthropic_messages_config(
model=model, provider=provider_enum
)
)
if anthropic_config is not None:
verbose_logger.debug(
f"WebSearchInterception: Skipping short-circuit for {provider_str} "
"(provider has native Anthropic Messages support, using agentic loop)"
)
return None
except (ValueError, Exception):
pass # unknown provider enum → safe to short-circuit
# All tools must be web search tools
if not all(is_web_search_tool(t) for t in tools):
return None

View file

@ -205,11 +205,18 @@ async def anthropic_messages(
# Extract modified parameters
tools = request_kwargs.pop("tools", tools)
stream = request_kwargs.pop("stream", stream)
# Propagate the provider derived inside pre-request hooks, if not already set
# Propagate the provider derived inside pre-request hooks, if not already set.
# The litellm_params dict may have been overwritten by **kwargs in
# _execute_pre_request_hooks, so fall back to get_llm_provider() if needed.
if not custom_llm_provider:
custom_llm_provider = request_kwargs.get("litellm_params", {}).get(
"custom_llm_provider"
)
if not custom_llm_provider:
try:
_, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model)
except Exception:
pass
# Remove litellm_params from kwargs (only needed for hooks)
request_kwargs.pop("litellm_params", None)
# Merge back any other modifications

View file

@ -114,6 +114,29 @@ class TestTryShortCircuitSearch:
assert result is None
@pytest.mark.asyncio
async def test_does_not_short_circuit_bedrock(self):
"""Bedrock has native agentic loop support → NOT short-circuited.
Providers with a BaseAnthropicMessagesConfig (bedrock, vertex_ai, etc.)
use the agentic loop which includes a follow-up LLM synthesis step.
The short-circuit must not fire for them.
"""
logger = WebSearchInterceptionLogger(
enabled_providers=["bedrock", "github_copilot"]
)
result = await logger.try_short_circuit_search(
model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
messages=[{"role": "user", "content": "Search for something"}],
tools=[
{"type": "web_search_20250305", "name": "web_search", "max_uses": 8}
],
custom_llm_provider="bedrock",
)
assert result is None
@pytest.mark.asyncio
async def test_does_not_short_circuit_no_messages(self):
"""Empty messages → NOT short-circuited"""