From a8e76594d0b3f14b87874b9a2550525933ed4447 Mon Sep 17 00:00:00 2001 From: skgandikota Date: Mon, 27 Apr 2026 22:53:09 +0100 Subject: [PATCH] fix(websearch_interception): preserve custom api_base/api_key in follow-up request When the websearch_interception agentic loop runs for a deployment with a custom api_base/api_key, the follow-up request after web-search results was sent to the provider's default api_base using the env-var API key (e.g. ANTHROPIC_API_KEY), causing 401 responses and a deployment cooldown cascade. Root cause: 1. anthropic_messages_handler stored only model + custom_llm_provider in logging_obj.model_call_details['agentic_loop_params'], dropping the dynamic_api_key / dynamic_api_base resolved by get_llm_provider(). 2. The websearch follow-up call in WebSearchInterceptionLogger built kwargs_for_followup without re-injecting api_key / api_base, so anthropic_messages.acreate fell back to env vars + default api_base. Fix: - Persist dynamic_api_key / dynamic_api_base into agentic_loop_params when get_llm_provider() returns them. - In _build_anthropic_request_patch, propagate those values into the follow-up kwargs (without overriding caller-supplied values). - Extend AgenticLoopParams TypedDict with the new optional fields. Tests added (tests/test_litellm/...): - Handler test: asserts agentic_loop_params contains api_key / api_base when get_llm_provider returns them, and omits those keys when None. - WebSearch test: asserts the follow-up request_patch.kwargs carry the api_key / api_base from agentic_loop_params, and that explicit caller kwargs win over agentic_loop_params copies. Fixes #26389 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../websearch_interception/handler.py | 11 ++ .../messages/handler.py | 13 +- litellm/types/utils.py | 11 ++ .../test_websearch_interception_handler.py | 117 ++++++++++++++++++ ...erimental_pass_through_messages_handler.py | 88 +++++++++++++ 5 files changed, 239 insertions(+), 1 deletion(-) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 41618c72627..1255f0467af 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -851,6 +851,17 @@ class WebSearchInterceptionLogger(CustomLogger): "agentic_loop_params", {} ) full_model_name = agentic_params.get("model", model) + # Preserve deployment-specific credentials in the follow-up + # request. Without these, the follow-up call falls back to + # provider env vars (e.g. ANTHROPIC_API_KEY) and the default + # public api_base, causing 401s and deployment cooldown + # cascades for routed deployments (#26389). + agentic_api_key = agentic_params.get("api_key") + agentic_api_base = agentic_params.get("api_base") + if agentic_api_key is not None and "api_key" not in kwargs_for_followup: + kwargs_for_followup["api_key"] = agentic_api_key + if agentic_api_base is not None and "api_base" not in kwargs_for_followup: + kwargs_for_followup["api_base"] = agentic_api_base verbose_logger.debug( "WebSearchInterception: Built anthropic request patch " "[call_id=%s model=%s messages=%d searches=%d]", diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 0c59e812e0b..f05a49f3e96 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -368,10 +368,21 @@ def anthropic_messages_handler( # Store agentic loop params in logging object for agentic hooks # This provides original request context needed for follow-up calls if litellm_logging_obj is not None: - litellm_logging_obj.model_call_details["agentic_loop_params"] = { + agentic_loop_params: Dict[str, Any] = { "model": original_model, "custom_llm_provider": custom_llm_provider, } + # Preserve the deployment-specific credentials resolved by + # get_llm_provider() so agentic follow-up requests (e.g. websearch + # interception) target the same api_base / api_key as the initial + # call, instead of falling back to provider env vars (#26389). + if dynamic_api_key is not None: + agentic_loop_params["api_key"] = dynamic_api_key + if dynamic_api_base is not None: + agentic_loop_params["api_base"] = dynamic_api_base + litellm_logging_obj.model_call_details["agentic_loop_params"] = ( + agentic_loop_params + ) # Check if stream was converted for WebSearch interception # This is set in the async wrapper above when stream=True is converted to stream=False diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a212d56c1ae..5f66ba1fd0a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -164,6 +164,17 @@ class AgenticLoopParams(TypedDict, total=False): custom_llm_provider: str """The LLM provider name (e.g., 'bedrock', 'anthropic')""" + api_key: str + """Deployment-specific API key resolved by get_llm_provider() (optional). + + Stored so agentic follow-up calls (e.g. websearch interception) reuse the + same credentials as the initial request instead of falling back to + provider env vars. + """ + + api_base: str + """Deployment-specific API base URL resolved by get_llm_provider() (optional).""" + class ModelInfoBase(ProviderSpecificModelInfo, total=False): key: Required[str] # the key in litellm.model_cost which is returned diff --git a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py index 10951265115..960c59e99c7 100644 --- a/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py +++ b/tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py @@ -380,3 +380,120 @@ async def test_deployment_hook_converts_stream_and_logging_obj_syncs(): logging_obj.stream = _hook_stream assert logging_obj.stream is False + + +@pytest.mark.asyncio +async def test_followup_request_preserves_custom_api_base_and_api_key(): + """ + Regression test for #26389. + + When the initial request was routed to a deployment with custom + api_base / api_key, the websearch interception follow-up call must + preserve those credentials. Without this, the follow-up falls back + to provider env vars and the default api_base, causing 401 and + deployment cooldown cascade. + """ + logger = WebSearchInterceptionLogger(enabled_providers=["anthropic"]) + logger._execute_search = AsyncMock( # type: ignore + return_value="Title: LiteLLM\nURL: docs\nSnippet: test" + ) + + tools_dict = { + "tool_calls": [ + { + "id": "toolu_xyz", + "type": "tool_use", + "name": "litellm_web_search", + "input": {"query": "what is litellm"}, + } + ], + "response_format": "anthropic", + } + logging_obj = MagicMock() + logging_obj.model_call_details = { + "agentic_loop_params": { + "model": "anthropic/claude-3-5-sonnet", + "custom_llm_provider": "anthropic", + "api_key": "deployment-specific-key", + "api_base": "https://my-proxy.example.com/v1", + } + } + + plan = await logger.async_build_agentic_loop_plan( + tools=tools_dict, + model="claude-3-5-sonnet", + messages=[{"role": "user", "content": "search"}], + response=None, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={ + "max_tokens": 1024, + "tools": [{"name": "litellm_web_search"}], + }, + logging_obj=logging_obj, + stream=False, + kwargs={"temperature": 0.2}, + ) + + assert plan.run_agentic_loop is True + assert plan.request_patch is not None + followup_kwargs = plan.request_patch.kwargs + assert followup_kwargs.get("api_key") == "deployment-specific-key" + assert followup_kwargs.get("api_base") == "https://my-proxy.example.com/v1" + + +@pytest.mark.asyncio +async def test_followup_request_does_not_override_user_provided_api_credentials(): + """ + If the caller already explicitly passed api_key / api_base in kwargs, + those values should win over the agentic_loop_params copies (caller + intent is preserved). + """ + logger = WebSearchInterceptionLogger(enabled_providers=["anthropic"]) + logger._execute_search = AsyncMock( # type: ignore + return_value="Title: LiteLLM\nURL: docs\nSnippet: test" + ) + + tools_dict = { + "tool_calls": [ + { + "id": "toolu_abc", + "type": "tool_use", + "name": "litellm_web_search", + "input": {"query": "x"}, + } + ], + "response_format": "anthropic", + } + logging_obj = MagicMock() + logging_obj.model_call_details = { + "agentic_loop_params": { + "model": "anthropic/claude-3-5-sonnet", + "custom_llm_provider": "anthropic", + "api_key": "from-agentic-params", + "api_base": "https://from-agentic-params.example.com/v1", + } + } + + plan = await logger.async_build_agentic_loop_plan( + tools=tools_dict, + model="claude-3-5-sonnet", + messages=[{"role": "user", "content": "search"}], + response=None, + anthropic_messages_provider_config=None, + anthropic_messages_optional_request_params={ + "max_tokens": 1024, + "tools": [{"name": "litellm_web_search"}], + }, + logging_obj=logging_obj, + stream=False, + kwargs={ + "temperature": 0.2, + "api_key": "explicit-caller-key", + "api_base": "https://explicit-caller.example.com/v1", + }, + ) + + assert plan.request_patch is not None + followup_kwargs = plan.request_patch.kwargs + assert followup_kwargs.get("api_key") == "explicit-caller-key" + assert followup_kwargs.get("api_base") == "https://explicit-caller.example.com/v1" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 33628e1d19d..cd9589eaa7d 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -499,3 +499,91 @@ class TestThinkingSummaryPreservation: assert result == { "reasoning_effort": {"effort": "medium", "summary": "concise"} } + + +def test_agentic_loop_params_preserves_dynamic_api_key_and_api_base(): + """ + Regression test for #26389. + + When get_llm_provider() returns deployment-specific dynamic_api_key / + dynamic_api_base, those values must be stored in + logging_obj.model_call_details["agentic_loop_params"] so agentic + follow-up calls (e.g. websearch interception) reuse the same + credentials instead of falling back to provider env vars. + """ + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + anthropic_messages_handler, + ) + + logging_obj = MagicMock() + logging_obj.model_call_details = {} + + with ( + patch( + "litellm.get_llm_provider", + return_value=( + "claude-3-5-sonnet", + "anthropic", + "deployment-specific-key", + "https://my-proxy.example.com/v1", + ), + ), + patch("litellm.completion", return_value="test-response"), + ): + try: + anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "hi"}], + model="anthropic/claude-3-5-sonnet", + custom_llm_provider="anthropic", + api_key="deployment-specific-key", + api_base="https://my-proxy.example.com/v1", + litellm_logging_obj=logging_obj, + ) + except (ValueError, TypeError, AttributeError): + pass + + agentic_loop_params = logging_obj.model_call_details.get("agentic_loop_params") + assert agentic_loop_params is not None + assert agentic_loop_params["model"] == "anthropic/claude-3-5-sonnet" + assert agentic_loop_params["custom_llm_provider"] == "anthropic" + assert agentic_loop_params["api_key"] == "deployment-specific-key" + assert agentic_loop_params["api_base"] == "https://my-proxy.example.com/v1" + + +def test_agentic_loop_params_omits_keys_when_dynamic_values_are_none(): + """ + When get_llm_provider() returns no dynamic_api_key / dynamic_api_base + (typical for default provider config), the agentic_loop_params dict + should NOT contain api_key / api_base keys, so downstream code can + fall back to its existing resolution logic. + """ + from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( + anthropic_messages_handler, + ) + + logging_obj = MagicMock() + logging_obj.model_call_details = {} + + with ( + patch( + "litellm.get_llm_provider", + return_value=("claude-3-5-sonnet", "anthropic", None, None), + ), + patch("litellm.completion", return_value="test-response"), + ): + try: + anthropic_messages_handler( + max_tokens=100, + messages=[{"role": "user", "content": "hi"}], + model="anthropic/claude-3-5-sonnet", + custom_llm_provider="anthropic", + litellm_logging_obj=logging_obj, + ) + except (ValueError, TypeError, AttributeError): + pass + + agentic_loop_params = logging_obj.model_call_details.get("agentic_loop_params") + assert agentic_loop_params is not None + assert "api_key" not in agentic_loop_params + assert "api_base" not in agentic_loop_params