diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index a38b8bed595..ece61600154 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -37,6 +37,7 @@ from litellm.integrations.websearch_interception.transformation import ( from litellm.litellm_core_utils.agentic_loop_settings import ( validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.types.integrations.custom_logger import ( CHAT_COMPLETION_AGENTIC_SURFACE, @@ -1251,9 +1252,10 @@ class WebSearchInterceptionLogger(CustomLogger): } } - full_model_name = model - if "/" not in model and isinstance(kwargs.get("custom_llm_provider"), str): - full_model_name = f"{kwargs['custom_llm_provider']}/{model}" + custom_llm_provider: Final = kwargs.get("custom_llm_provider") + full_model_name: Final = qualify_provider_stripped_model( + model, custom_llm_provider if isinstance(custom_llm_provider, str) else "" + ) verbose_logger.debug( "WebSearchInterception: Built responses request patch model=%s input_items=%d searches=%d", @@ -1833,7 +1835,7 @@ class WebSearchInterceptionLogger(CustomLogger): messages: list[dict], tool_calls: list[dict], optional_params: dict, - kwargs: dict, + kwargs: dict[str, object], response_format: str = "openai", ) -> AgenticLoopRequestPatch: """Execute litellm.search() and build chat-completion rerun patch.""" @@ -1919,11 +1921,10 @@ class WebSearchInterceptionLogger(CustomLogger): k: v for k, v in kwargs.items() if not k.startswith("_websearch_interception") and k not in internal_params } - full_model_name = model - if "custom_llm_provider" in kwargs: - custom_llm_provider: Final = kwargs["custom_llm_provider"] - if not model.startswith(custom_llm_provider) and "/" not in model: - full_model_name = f"{custom_llm_provider}/{model}" + custom_llm_provider: Final = kwargs.get("custom_llm_provider") + full_model_name: Final = qualify_provider_stripped_model( + model, custom_llm_provider if isinstance(custom_llm_provider, str) else "" + ) verbose_logger.debug( "WebSearchInterception: Built chat completion request patch model=%s messages=%d", diff --git a/litellm/litellm_core_utils/chat_completion_agentic_loop.py b/litellm/litellm_core_utils/chat_completion_agentic_loop.py index 628486f377f..2ba47cbb400 100644 --- a/litellm/litellm_core_utils/chat_completion_agentic_loop.py +++ b/litellm/litellm_core_utils/chat_completion_agentic_loop.py @@ -15,6 +15,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import ( DEFAULT_MAX_AGENTIC_LOOPS, validated_max_agentic_loops, ) +from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject from litellm.llms.base_llm.base_model_iterator import MockResponseIterator from litellm.types.integrations.custom_logger import ( @@ -182,9 +183,7 @@ async def _execute_chat_completion_agentic_plan( if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = {**optional_params, **patch.optional_params} if patch.tools is not None: diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 00c6ce97e78..3c59a0710d9 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -34,6 +34,48 @@ def is_codex_user_agent(user_agent: str) -> bool: return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent)) +def qualify_provider_stripped_model(model: str, custom_llm_provider: str) -> str: + """Put the provider prefix back on a provider-stripped model.""" + if not custom_llm_provider or model.startswith(f"{custom_llm_provider}/"): + return model + if _already_routes_to(model, custom_llm_provider): + return model + return f"{custom_llm_provider}/{model}" + + +def _already_routes_to(model: str, custom_llm_provider: str) -> bool: + from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider + + leading_segment: Final = model.split("/", 1)[0] + if leading_segment == model or not names_a_provider(leading_segment): + return False + return inferred_provider(model) == custom_llm_provider + + +def names_a_provider(candidate: str) -> bool: + """Whether a model name's leading segment is a provider rather than part of the model.""" + import litellm + from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider + from litellm.llms.openai_like.json_loader import JSONProviderRegistry + + provider_names: Final = {getattr(provider, "value", provider) for provider in litellm.provider_list} + return ( + candidate in provider_names + or JSONProviderRegistry.exists(candidate) + or is_registered_custom_provider(candidate) + ) + + +def qualify_agentic_followup_model(patch_model: str | None, model: str, custom_llm_provider: str) -> str: + """Resolve the model an agentic follow-up re-dispatches, keeping a hook's own provider.""" + if not patch_model or patch_model == model: + return qualify_provider_stripped_model(model, custom_llm_provider) + leading_segment: Final = patch_model.split("/", 1)[0] + if leading_segment != patch_model and names_a_provider(leading_segment): + return patch_model + return qualify_provider_stripped_model(patch_model, custom_llm_provider) + + def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None: """ Safely divide seconds by denominator, handling zero division. diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ef009e70137..6e3fb8137e9 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -49,7 +49,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( SUBTITLE_RESPONSE_FORMATS, synthesize_subtitle_document, ) -from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params +from litellm.litellm_core_utils.core_helpers import ( + as_str_mapping, + qualify_agentic_followup_model, + set_provider_response_headers_in_hidden_params, +) from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header @@ -6169,9 +6173,7 @@ class BaseLLMHTTPHandler: if patch.messages is None: raise ValueError("Agentic loop plan missing patched messages") - full_model_name = patch.model or model - if "/" not in full_model_name: - full_model_name = f"{custom_llm_provider}/{full_model_name}" + full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider) optional_params_for_followup: Final = dict(optional_params) optional_params_for_followup.update(patch.optional_params) diff --git a/tests/unit/integrations/websearch_interception/test_websearch_responses.py b/tests/unit/integrations/websearch_interception/test_websearch_responses.py index 00b7299cc31..feb3b0dd987 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_responses.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_responses.py @@ -7,16 +7,21 @@ builds a Responses-format follow-up request. """ from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest +import litellm + from litellm.integrations.websearch_interception.handler import ( WebSearchInterceptionLogger, ) from litellm.types.integrations.custom_logger import ( RESPONSES_AGENTIC_SURFACE, ) +from litellm.llms.base_llm.search.transformation import SearchResponse +from litellm.proxy import proxy_server from litellm.types.utils import CallTypes, LlmProviders @@ -252,6 +257,50 @@ async def test_build_responses_plan_produces_responses_input(): assert patch_obj.model == "openai/gpt-4o" +@pytest.mark.asyncio +async def test_build_responses_plan_keeps_the_provider_on_a_sub_path_model(monkeypatch: pytest.MonkeyPatch) -> None: + """#38829 on the Responses surface: a provider-stripped model whose remainder still holds a + slash was left unqualified, so the follow-up could not resolve a provider.""" + logger: Final = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.BEDROCK]) + monkeypatch.setattr(proxy_server, "llm_router", None) + monkeypatch.setattr(litellm, "asearch", AsyncMock(return_value=SearchResponse(object="search", results=[]))) + + tools_dict: Final = { + "tool_calls": [ + { + "id": "fc_1", + "call_id": "fc_1", + "type": "function_call", + "name": "litellm_web_search", + "arguments": '{"query": "latest ai news"}', + "input": {"query": "latest ai news"}, + } + ], + "tool_type": "websearch", + "provider": "bedrock", + "response_format": "responses", + } + + plan: Final = await logger.async_build_responses_agentic_loop_plan( + tools=tools_dict, + model="mantle/anthropic.claude-sonnet-5", + messages=[{"role": "user", "content": "What's the latest AI news?"}], + response=_responses_output_with_web_search(), + optional_params={"tools": [{"type": "function", "name": "litellm_web_search"}]}, + logging_obj=MagicMock(), + stream=False, + kwargs={ + "custom_llm_provider": "bedrock", + "_agentic_loop_api_surface": RESPONSES_AGENTIC_SURFACE, + }, + ) + + patch_obj: Final = plan.request_patch + assert patch_obj is not None + assert patch_obj.model == "bedrock/mantle/anthropic.claude-sonnet-5" + litellm.asearch.assert_awaited_once() + + @pytest.mark.asyncio async def test_deployment_hook_converts_native_responses_web_search_tool(): """async_pre_call_deployment_hook converts a native Responses web_search tool diff --git a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py index 3cbc3591009..6167e91c37f 100644 --- a/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py +++ b/tests/unit/litellm_core_utils/test_chat_completion_agentic_loop.py @@ -364,6 +364,49 @@ async def test_dispatcher_runs_followup_with_incremented_depth_and_patched_messa assert logger.cleanup_calls == 1 +@pytest.mark.asyncio +@pytest.mark.parametrize( + "model,custom_llm_provider,routed_model", + [ + ("cohere/command-a-03-2025", "cohere_chat", "command-a-03-2025"), + ("anthropic/claude-2", "anthropic_text", "claude-2"), + ], +) +async def test_sdk_followup_keeps_a_model_whose_prefix_already_routes_to_its_provider( + model: str, custom_llm_provider: str, routed_model: str +) -> None: + plan: Final = AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch(messages=_patched_messages()), + ) + gate: Final = _GateOnlyLogger(plan=plan, tool_calls={"tool_calls": [{"id": "call_abc"}]}) + logging_obj: Final = Logging( + model=model, + messages=[{"role": "user", "content": "what is 6*7?"}], + stream=False, + call_type="acompletion", + start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc), + litellm_call_id="call-prefix", + function_id="fn-prefix", + dynamic_success_callbacks=[gate], + ) + + followup: Final = await maybe_run_chat_completion_agentic_loop( + response=_tool_call_model_response(), + model=model, + messages=[{"role": "user", "content": "what is 6*7?"}], + optional_params={}, + kwargs={"mock_response": "done"}, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + stream=False, + ) + + assert isinstance(followup, ModelResponse) + assert followup.model == routed_model + assert followup._hidden_params["custom_llm_provider"] == custom_llm_provider + + @pytest.mark.asyncio async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_request_kwargs( restore_callbacks, diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index d66c37ba8c2..479b32cd7d7 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -26,6 +26,8 @@ from litellm.litellm_core_utils.core_helpers import ( normalize_drop_params, preserve_upstream_non_openai_attributes, process_response_headers, + qualify_agentic_followup_model, + qualify_provider_stripped_model, reconstruct_model_name, redact_nested_match_and_regex_keys, set_provider_response_headers_in_hidden_params, @@ -636,6 +638,87 @@ class TestProviderResponseHeadersInHiddenParams: assert "additional_headers" not in sibling._hidden_params +@pytest.mark.parametrize( + "model,provider,expected", + [ + ("mantle/anthropic.claude-sonnet-5", "bedrock", "bedrock/mantle/anthropic.claude-sonnet-5"), + ("invoke/anthropic.claude-v2", "bedrock", "bedrock/invoke/anthropic.claude-v2"), + ("openai/gpt-4o", "openrouter", "openrouter/openai/gpt-4o"), + ("gpt-4o", "openai", "openai/gpt-4o"), + ("claude-sonnet-4-5", "anthropic", "anthropic/claude-sonnet-4-5"), + ], +) +def test_the_provider_prefix_is_restored(model: str, provider: str, expected: str) -> None: + assert qualify_provider_stripped_model(model, provider) == expected + + +@pytest.mark.parametrize( + "model,provider", + [ + ("bedrock/mantle/anthropic.claude-sonnet-5", "bedrock"), + ("openai/gpt-4o", "openai"), + ("cohere/command-a-03-2025", "cohere_chat"), + ("ai21/jamba-1.5-large", "ai21_chat"), + ("anthropic/claude-2", "anthropic_text"), + ], +) +def test_an_already_qualified_model_is_left_alone(model: str, provider: str) -> None: + assert qualify_provider_stripped_model(model, provider) == model + + +def test_a_provider_that_only_shares_a_prefix_is_still_qualified() -> None: + assert qualify_provider_stripped_model("openai_like/foo", "openai") == "openai/openai_like/foo" + + +def test_no_provider_leaves_the_model_untouched() -> None: + assert qualify_provider_stripped_model("gpt-4o", "") == "gpt-4o" + + +def test_a_cross_provider_patched_model_is_dispatched_as_the_hook_asked() -> None: + assert qualify_agentic_followup_model("anthropic/claude-sonnet-4-5", "gpt-4o", "openai") == ( + "anthropic/claude-sonnet-4-5" + ) + + +def test_a_bare_patched_model_takes_the_request_provider() -> None: + assert qualify_agentic_followup_model("gpt-4o-mini", "gpt-4o", "openai") == "openai/gpt-4o-mini" + + +def test_a_sub_path_request_model_keeps_its_provider() -> None: + assert qualify_agentic_followup_model(None, "mantle/anthropic.claude-sonnet-5", "bedrock") == ( + "bedrock/mantle/anthropic.claude-sonnet-5" + ) + + +def test_an_unpatched_ordinary_model_takes_the_request_provider() -> None: + assert qualify_agentic_followup_model(None, "gpt-4o", "openai") == "openai/gpt-4o" + + +def test_a_patched_model_already_holding_the_request_provider_is_left_alone() -> None: + assert qualify_agentic_followup_model("openai/gpt-4o", "gpt-4o", "openai") == "openai/gpt-4o" + + +@pytest.mark.parametrize("patched", ["", None]) +def test_a_patch_without_a_model_falls_back_to_the_request_model(patched: str | None) -> None: + assert qualify_agentic_followup_model(patched, "gpt-4o", "openai") == "openai/gpt-4o" + + +def test_a_patched_sub_path_model_still_takes_the_request_provider() -> None: + assert qualify_agentic_followup_model("mantle/anthropic.claude-sonnet-5", "gpt-4o", "bedrock") == ( + "bedrock/mantle/anthropic.claude-sonnet-5" + ) + + +def test_a_patched_model_on_a_json_configured_provider_reaches_that_provider() -> None: + assert qualify_agentic_followup_model("aihubmix/gpt-4o", "gpt-4o", "openai") == "aihubmix/gpt-4o" + + +def test_a_patch_echoing_the_request_model_keeps_the_request_provider() -> None: + assert qualify_agentic_followup_model("openai/gpt-4o", "openai/gpt-4o", "openrouter") == ( + "openrouter/openai/gpt-4o" + ) + + def test_spend_metadata_merge_survives_the_logging_thread_inserting_into_the_request_metadata() -> None: metadata: Final[dict[str, object]] = {} diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 297e64629a8..b8f9edcdc42 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -5545,6 +5545,67 @@ async def test_anthropic_messages_dropped_safeguards_send_neither_the_field_nor_ assert "dangerous-tool-use-2026-09-03" not in sent.headers.get("anthropic-beta", ""), dict(sent.headers) +class TestAgenticFollowUpKeepsTheProviderPrefix: + """#38829: the follow-up must re-dispatch a model litellm.acompletion can route.""" + + @staticmethod + def _plan(): + from litellm.types.integrations.custom_logger import ( + AgenticLoopPlan, + AgenticLoopRequestPatch, + ) + + return AgenticLoopPlan( + run_agentic_loop=True, + request_patch=AgenticLoopRequestPatch(messages=[{"role": "user", "content": "hi"}]), + ) + + async def _run_followup(self, model: str, custom_llm_provider: str, patched_model: str | None = None): + from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler + + plan = self._plan() + if patched_model is not None: + plan.request_patch.model = patched_model + + return await BaseLLMHTTPHandler()._execute_chat_completion_agentic_plan( + plan=plan, + model=model, + messages=[{"role": "user", "content": "hi"}], + optional_params={"mock_response": "ok from followup"}, + kwargs={}, + custom_llm_provider=custom_llm_provider, + depth=0, + max_loops=2, + fingerprints=[], + fingerprint="fp", + ) + + @pytest.mark.asyncio + async def test_a_sub_path_model_still_resolves_a_provider(self): + response = await self._run_followup("mantle/anthropic.claude-sonnet-5", "bedrock") + + assert response.choices[0].message.content == "ok from followup" + + @pytest.mark.asyncio + async def test_an_ordinary_model_still_resolves_a_provider(self): + response = await self._run_followup("gpt-4o-mini", "openai") + + assert response.choices[0].message.content == "ok from followup" + + @pytest.mark.asyncio + async def test_a_hook_that_patches_another_providers_model_reaches_that_provider(self): + response = await self._run_followup("gpt-4o", "openai", patched_model="anthropic/claude-sonnet-4-5") + + assert response.model == "claude-sonnet-4-5" + assert response._hidden_params["custom_llm_provider"] == "anthropic" + + @pytest.mark.asyncio + async def test_a_hook_echoing_an_org_style_model_stays_on_the_request_provider(self): + response = await self._run_followup("openai/gpt-4o", "openrouter", patched_model="openai/gpt-4o") + + assert response._hidden_params["custom_llm_provider"] == "openrouter" + + @pytest.mark.asyncio @pytest.mark.parametrize("stream", [False, True]) @pytest.mark.parametrize("environment_base", [None, "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"])