mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-08 03:08:45 +00:00
fix: guard short-circuit against providers with native agentic loop
- Skip short-circuit for providers that have a BaseAnthropicMessagesConfig (bedrock, vertex_ai, azure_ai, anthropic) — they use the agentic loop which includes a follow-up LLM synthesis step. Short-circuiting would return raw search text instead of an LLM-synthesized answer. - Add fallback to litellm.get_llm_provider() for custom_llm_provider derivation when litellm_params is overwritten by kwargs. - Add test for bedrock guard. Addresses Greptile review comments #3 and #4.
This commit is contained in:
parent
141ad04955
commit
32cb6f0cd9
3 changed files with 54 additions and 1 deletions
|
|
@ -29,6 +29,7 @@ from litellm.types.integrations.websearch_interception import (
|
|||
WebSearchInterceptionConfig,
|
||||
)
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
|
||||
class WebSearchInterceptionLogger(CustomLogger):
|
||||
|
|
@ -106,6 +107,28 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
):
|
||||
return None
|
||||
|
||||
# Only short-circuit for providers without native Anthropic Messages
|
||||
# support. Providers that have a BaseAnthropicMessagesConfig (bedrock,
|
||||
# vertex_ai, azure_ai, anthropic) already use the agentic loop, which
|
||||
# includes a follow-up LLM call to synthesize the answer from search
|
||||
# results. Short-circuiting those would skip that synthesis step and
|
||||
# return raw search text — a regression for existing users.
|
||||
try:
|
||||
provider_enum = LlmProviders(provider_str)
|
||||
anthropic_config = (
|
||||
ProviderConfigManager.get_provider_anthropic_messages_config(
|
||||
model=model, provider=provider_enum
|
||||
)
|
||||
)
|
||||
if anthropic_config is not None:
|
||||
verbose_logger.debug(
|
||||
f"WebSearchInterception: Skipping short-circuit for {provider_str} "
|
||||
"(provider has native Anthropic Messages support, using agentic loop)"
|
||||
)
|
||||
return None
|
||||
except (ValueError, Exception):
|
||||
pass # unknown provider enum → safe to short-circuit
|
||||
|
||||
# All tools must be web search tools
|
||||
if not all(is_web_search_tool(t) for t in tools):
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -205,11 +205,18 @@ async def anthropic_messages(
|
|||
# Extract modified parameters
|
||||
tools = request_kwargs.pop("tools", tools)
|
||||
stream = request_kwargs.pop("stream", stream)
|
||||
# Propagate the provider derived inside pre-request hooks, if not already set
|
||||
# Propagate the provider derived inside pre-request hooks, if not already set.
|
||||
# The litellm_params dict may have been overwritten by **kwargs in
|
||||
# _execute_pre_request_hooks, so fall back to get_llm_provider() if needed.
|
||||
if not custom_llm_provider:
|
||||
custom_llm_provider = request_kwargs.get("litellm_params", {}).get(
|
||||
"custom_llm_provider"
|
||||
)
|
||||
if not custom_llm_provider:
|
||||
try:
|
||||
_, custom_llm_provider, _, _ = litellm.get_llm_provider(model=model)
|
||||
except Exception:
|
||||
pass
|
||||
# Remove litellm_params from kwargs (only needed for hooks)
|
||||
request_kwargs.pop("litellm_params", None)
|
||||
# Merge back any other modifications
|
||||
|
|
|
|||
|
|
@ -114,6 +114,29 @@ class TestTryShortCircuitSearch:
|
|||
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_short_circuit_bedrock(self):
|
||||
"""Bedrock has native agentic loop support → NOT short-circuited.
|
||||
|
||||
Providers with a BaseAnthropicMessagesConfig (bedrock, vertex_ai, etc.)
|
||||
use the agentic loop which includes a follow-up LLM synthesis step.
|
||||
The short-circuit must not fire for them.
|
||||
"""
|
||||
logger = WebSearchInterceptionLogger(
|
||||
enabled_providers=["bedrock", "github_copilot"]
|
||||
)
|
||||
|
||||
result = await logger.try_short_circuit_search(
|
||||
model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
messages=[{"role": "user", "content": "Search for something"}],
|
||||
tools=[
|
||||
{"type": "web_search_20250305", "name": "web_search", "max_uses": 8}
|
||||
],
|
||||
custom_llm_provider="bedrock",
|
||||
)
|
||||
|
||||
assert result is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_does_not_short_circuit_no_messages(self):
|
||||
"""Empty messages → NOT short-circuited"""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue