scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

fix(websearch_interception): preserve custom api_base/api_key in follow-up request

Browse source 
When the websearch_interception agentic loop runs for a deployment with a custom api_base/api_key, the follow-up request after web-search results was sent to the provider's default api_base using the env-var API key (e.g. ANTHROPIC_API_KEY), causing 401 responses and a deployment cooldown cascade.

Root cause:
  1. anthropic_messages_handler stored only model + custom_llm_provider in
     logging_obj.model_call_details['agentic_loop_params'], dropping the
     dynamic_api_key / dynamic_api_base resolved by get_llm_provider().
  2. The websearch follow-up call in WebSearchInterceptionLogger built
     kwargs_for_followup without re-injecting api_key / api_base, so
     anthropic_messages.acreate fell back to env vars + default api_base.

Fix:
  - Persist dynamic_api_key / dynamic_api_base into agentic_loop_params
    when get_llm_provider() returns them.
  - In _build_anthropic_request_patch, propagate those values into the
    follow-up kwargs (without overriding caller-supplied values).
  - Extend AgenticLoopParams TypedDict with the new optional fields.

Tests added (tests/test_litellm/...):
  - Handler test: asserts agentic_loop_params contains api_key / api_base
    when get_llm_provider returns them, and omits those keys when None.
  - WebSearch test: asserts the follow-up request_patch.kwargs carry the
    api_key / api_base from agentic_loop_params, and that explicit caller
    kwargs win over agentic_loop_params copies.

Fixes #26389

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
skgandikota 2026-04-27 22:53:09 +01:00
parent 82dacfb746
commit a8e76594d0
No known key found for this signature in database
GPG key ID: 9C8C0949390E528F
5 changed files with 239 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
litellm/integrations/websearch_interception/handler.py
View file

@ -851,6 +851,17 @@ class WebSearchInterceptionLogger(CustomLogger):
"agentic_loop_params", {}
)
full_model_name = agentic_params.get("model", model)
# Preserve deployment-specific credentials in the follow-up
# request. Without these, the follow-up call falls back to
# provider env vars (e.g. ANTHROPIC_API_KEY) and the default
# public api_base, causing 401s and deployment cooldown
# cascades for routed deployments (#26389).
agentic_api_key = agentic_params.get("api_key")
agentic_api_base = agentic_params.get("api_base")
if agentic_api_key is not None and "api_key" not in kwargs_for_followup:
kwargs_for_followup["api_key"] = agentic_api_key
if agentic_api_base is not None and "api_base" not in kwargs_for_followup:
kwargs_for_followup["api_base"] = agentic_api_base
verbose_logger.debug(
"WebSearchInterception: Built anthropic request patch "
"[call_id=%s model=%s messages=%d searches=%d]",

13
litellm/llms/anthropic/experimental_pass_through/messages/handler.py
View file

@ -368,10 +368,21 @@ def anthropic_messages_handler(
# Store agentic loop params in logging object for agentic hooks
# This provides original request context needed for follow-up calls
if litellm_logging_obj is not None:
litellm_logging_obj.model_call_details["agentic_loop_params"] = {
agentic_loop_params: Dict[str, Any] = {
"model": original_model,
"custom_llm_provider": custom_llm_provider,
}
# Preserve the deployment-specific credentials resolved by
# get_llm_provider() so agentic follow-up requests (e.g. websearch
# interception) target the same api_base / api_key as the initial
# call, instead of falling back to provider env vars (#26389).
if dynamic_api_key is not None:
agentic_loop_params["api_key"] = dynamic_api_key
if dynamic_api_base is not None:
agentic_loop_params["api_base"] = dynamic_api_base
litellm_logging_obj.model_call_details["agentic_loop_params"] = (
agentic_loop_params
)
# Check if stream was converted for WebSearch interception
# This is set in the async wrapper above when stream=True is converted to stream=False

11
litellm/types/utils.py
View file

@ -164,6 +164,17 @@ class AgenticLoopParams(TypedDict, total=False):
custom_llm_provider: str
"""The LLM provider name (e.g., 'bedrock', 'anthropic')"""
api_key: str
"""Deployment-specific API key resolved by get_llm_provider() (optional).
Stored so agentic follow-up calls (e.g. websearch interception) reuse the
same credentials as the initial request instead of falling back to
provider env vars.
"""
api_base: str
"""Deployment-specific API base URL resolved by get_llm_provider() (optional)."""
class ModelInfoBase(ProviderSpecificModelInfo, total=False):
key: Required[str] # the key in litellm.model_cost which is returned

117
tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py
View file

@ -380,3 +380,120 @@ async def test_deployment_hook_converts_stream_and_logging_obj_syncs():
logging_obj.stream = _hook_stream
assert logging_obj.stream is False
@pytest.mark.asyncio
async def test_followup_request_preserves_custom_api_base_and_api_key():
"""
Regression test for #26389.
When the initial request was routed to a deployment with custom
api_base / api_key, the websearch interception follow-up call must
preserve those credentials. Without this, the follow-up falls back
to provider env vars and the default api_base, causing 401 and
deployment cooldown cascade.
"""
logger = WebSearchInterceptionLogger(enabled_providers=["anthropic"])
logger._execute_search = AsyncMock( # type: ignore
return_value="Title: LiteLLM\nURL: docs\nSnippet: test"
)
tools_dict = {
"tool_calls": [
{
"id": "toolu_xyz",
"type": "tool_use",
"name": "litellm_web_search",
"input": {"query": "what is litellm"},
}
],
"response_format": "anthropic",
}
logging_obj = MagicMock()
logging_obj.model_call_details = {
"agentic_loop_params": {
"model": "anthropic/claude-3-5-sonnet",
"custom_llm_provider": "anthropic",
"api_key": "deployment-specific-key",
"api_base": "https://my-proxy.example.com/v1",
}
}
plan = await logger.async_build_agentic_loop_plan(
tools=tools_dict,
model="claude-3-5-sonnet",
messages=[{"role": "user", "content": "search"}],
response=None,
anthropic_messages_provider_config=None,
anthropic_messages_optional_request_params={
"max_tokens": 1024,
"tools": [{"name": "litellm_web_search"}],
},
logging_obj=logging_obj,
stream=False,
kwargs={"temperature": 0.2},
)
assert plan.run_agentic_loop is True
assert plan.request_patch is not None
followup_kwargs = plan.request_patch.kwargs
assert followup_kwargs.get("api_key") == "deployment-specific-key"
assert followup_kwargs.get("api_base") == "https://my-proxy.example.com/v1"
@pytest.mark.asyncio
async def test_followup_request_does_not_override_user_provided_api_credentials():
"""
If the caller already explicitly passed api_key / api_base in kwargs,
those values should win over the agentic_loop_params copies (caller
intent is preserved).
"""
logger = WebSearchInterceptionLogger(enabled_providers=["anthropic"])
logger._execute_search = AsyncMock( # type: ignore
return_value="Title: LiteLLM\nURL: docs\nSnippet: test"
)
tools_dict = {
"tool_calls": [
{
"id": "toolu_abc",
"type": "tool_use",
"name": "litellm_web_search",
"input": {"query": "x"},
}
],
"response_format": "anthropic",
}
logging_obj = MagicMock()
logging_obj.model_call_details = {
"agentic_loop_params": {
"model": "anthropic/claude-3-5-sonnet",
"custom_llm_provider": "anthropic",
"api_key": "from-agentic-params",
"api_base": "https://from-agentic-params.example.com/v1",
}
}
plan = await logger.async_build_agentic_loop_plan(
tools=tools_dict,
model="claude-3-5-sonnet",
messages=[{"role": "user", "content": "search"}],
response=None,
anthropic_messages_provider_config=None,
anthropic_messages_optional_request_params={
"max_tokens": 1024,
"tools": [{"name": "litellm_web_search"}],
},
logging_obj=logging_obj,
stream=False,
kwargs={
"temperature": 0.2,
"api_key": "explicit-caller-key",
"api_base": "https://explicit-caller.example.com/v1",
},
)
assert plan.request_patch is not None
followup_kwargs = plan.request_patch.kwargs
assert followup_kwargs.get("api_key") == "explicit-caller-key"
assert followup_kwargs.get("api_base") == "https://explicit-caller.example.com/v1"

88
tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
View file

@ -499,3 +499,91 @@ class TestThinkingSummaryPreservation:
assert result == {
"reasoning_effort": {"effort": "medium", "summary": "concise"}
}
def test_agentic_loop_params_preserves_dynamic_api_key_and_api_base():
"""
Regression test for #26389.
When get_llm_provider() returns deployment-specific dynamic_api_key /
dynamic_api_base, those values must be stored in
logging_obj.model_call_details["agentic_loop_params"] so agentic
follow-up calls (e.g. websearch interception) reuse the same
credentials instead of falling back to provider env vars.
"""
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
anthropic_messages_handler,
)
logging_obj = MagicMock()
logging_obj.model_call_details = {}
with (
patch(
"litellm.get_llm_provider",
return_value=(
"claude-3-5-sonnet",
"anthropic",
"deployment-specific-key",
"https://my-proxy.example.com/v1",
),
),
patch("litellm.completion", return_value="test-response"),
):
try:
anthropic_messages_handler(
max_tokens=100,
messages=[{"role": "user", "content": "hi"}],
model="anthropic/claude-3-5-sonnet",
custom_llm_provider="anthropic",
api_key="deployment-specific-key",
api_base="https://my-proxy.example.com/v1",
litellm_logging_obj=logging_obj,
)
except (ValueError, TypeError, AttributeError):
pass
agentic_loop_params = logging_obj.model_call_details.get("agentic_loop_params")
assert agentic_loop_params is not None
assert agentic_loop_params["model"] == "anthropic/claude-3-5-sonnet"
assert agentic_loop_params["custom_llm_provider"] == "anthropic"
assert agentic_loop_params["api_key"] == "deployment-specific-key"
assert agentic_loop_params["api_base"] == "https://my-proxy.example.com/v1"
def test_agentic_loop_params_omits_keys_when_dynamic_values_are_none():
"""
When get_llm_provider() returns no dynamic_api_key / dynamic_api_base
(typical for default provider config), the agentic_loop_params dict
should NOT contain api_key / api_base keys, so downstream code can
fall back to its existing resolution logic.
"""
from litellm.llms.anthropic.experimental_pass_through.messages.handler import (
anthropic_messages_handler,
)
logging_obj = MagicMock()
logging_obj.model_call_details = {}
with (
patch(
"litellm.get_llm_provider",
return_value=("claude-3-5-sonnet", "anthropic", None, None),
),
patch("litellm.completion", return_value="test-response"),
):
try:
anthropic_messages_handler(
max_tokens=100,
messages=[{"role": "user", "content": "hi"}],
model="anthropic/claude-3-5-sonnet",
custom_llm_provider="anthropic",
litellm_logging_obj=logging_obj,
)
except (ValueError, TypeError, AttributeError):
pass
agentic_loop_params = logging_obj.model_call_details.get("agentic_loop_params")
assert agentic_loop_params is not None
assert "api_key" not in agentic_loop_params
assert "api_base" not in agentic_loop_params