This commit is contained in:
Priyansh Nandwana 2026-10-28 07:44:12 +05:30 • committed by GitHub
commit f8ae64dcb4
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 296 additions and 16 deletions

View file

@ -37,6 +37,7 @@ from litellm.integrations.websearch_interception.transformation import (
from litellm.litellm_core_utils.agentic_loop_settings import (
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.types.integrations.custom_logger import (
CHAT_COMPLETION_AGENTIC_SURFACE,
@ -1251,9 +1252,10 @@ class WebSearchInterceptionLogger(CustomLogger):
}
}
full_model_name = model
if "/" not in model and isinstance(kwargs.get("custom_llm_provider"), str):
full_model_name = f"{kwargs['custom_llm_provider']}/{model}"
custom_llm_provider: Final = kwargs.get("custom_llm_provider")
full_model_name: Final = qualify_provider_stripped_model(
model, custom_llm_provider if isinstance(custom_llm_provider, str) else ""
)
verbose_logger.debug(
"WebSearchInterception: Built responses request patch model=%s input_items=%d searches=%d",
@ -1833,7 +1835,7 @@ class WebSearchInterceptionLogger(CustomLogger):
messages: list[dict],
tool_calls: list[dict],
optional_params: dict,
kwargs: dict,
kwargs: dict[str, object],
response_format: str = "openai",
) -> AgenticLoopRequestPatch:
"""Execute litellm.search() and build chat-completion rerun patch."""
@ -1919,11 +1921,10 @@ class WebSearchInterceptionLogger(CustomLogger):
k: v for k, v in kwargs.items() if not k.startswith("_websearch_interception") and k not in internal_params
}
full_model_name = model
if "custom_llm_provider" in kwargs:
custom_llm_provider: Final = kwargs["custom_llm_provider"]
if not model.startswith(custom_llm_provider) and "/" not in model:
full_model_name = f"{custom_llm_provider}/{model}"
custom_llm_provider: Final = kwargs.get("custom_llm_provider")
full_model_name: Final = qualify_provider_stripped_model(
model, custom_llm_provider if isinstance(custom_llm_provider, str) else ""
)
verbose_logger.debug(
"WebSearchInterception: Built chat completion request patch model=%s messages=%d",

View file

@ -15,6 +15,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
DEFAULT_MAX_AGENTIC_LOOPS,
validated_max_agentic_loops,
)
from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
from litellm.types.integrations.custom_logger import (
@ -182,9 +183,7 @@ async def _execute_chat_completion_agentic_plan(
if patch.messages is None:
raise ValueError("Agentic loop plan missing patched messages")
full_model_name = patch.model or model
if "/" not in full_model_name:
full_model_name = f"{custom_llm_provider}/{full_model_name}"
full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider)
optional_params_for_followup: Final = {**optional_params, **patch.optional_params}
if patch.tools is not None:

View file

@ -34,6 +34,48 @@ def is_codex_user_agent(user_agent: str) -> bool:
return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent))
def qualify_provider_stripped_model(model: str, custom_llm_provider: str) -> str:
"""Put the provider prefix back on a provider-stripped model."""
if not custom_llm_provider or model.startswith(f"{custom_llm_provider}/"):
return model
if _already_routes_to(model, custom_llm_provider):
return model
return f"{custom_llm_provider}/{model}"
def _already_routes_to(model: str, custom_llm_provider: str) -> bool:
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
leading_segment: Final = model.split("/", 1)[0]
if leading_segment == model or not names_a_provider(leading_segment):
return False
return inferred_provider(model) == custom_llm_provider
def names_a_provider(candidate: str) -> bool:
"""Whether a model name's leading segment is a provider rather than part of the model."""
import litellm
from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
provider_names: Final = {getattr(provider, "value", provider) for provider in litellm.provider_list}
return (
candidate in provider_names
or JSONProviderRegistry.exists(candidate)
or is_registered_custom_provider(candidate)
)
def qualify_agentic_followup_model(patch_model: str | None, model: str, custom_llm_provider: str) -> str:
"""Resolve the model an agentic follow-up re-dispatches, keeping a hook's own provider."""
if not patch_model or patch_model == model:
return qualify_provider_stripped_model(model, custom_llm_provider)
leading_segment: Final = patch_model.split("/", 1)[0]
if leading_segment != patch_model and names_a_provider(leading_segment):
return patch_model
return qualify_provider_stripped_model(patch_model, custom_llm_provider)
def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None:
"""
Safely divide seconds by denominator, handling zero division.

View file

@ -49,7 +49,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
SUBTITLE_RESPONSE_FORMATS,
synthesize_subtitle_document,
)
from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.core_helpers import (
as_str_mapping,
qualify_agentic_followup_model,
set_provider_response_headers_in_hidden_params,
)
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
@ -6169,9 +6173,7 @@ class BaseLLMHTTPHandler:
if patch.messages is None:
raise ValueError("Agentic loop plan missing patched messages")
full_model_name = patch.model or model
if "/" not in full_model_name:
full_model_name = f"{custom_llm_provider}/{full_model_name}"
full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider)
optional_params_for_followup: Final = dict(optional_params)
optional_params_for_followup.update(patch.optional_params)

View file

@ -7,16 +7,21 @@ builds a Responses-format follow-up request.
"""
from types import SimpleNamespace
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
import litellm
from litellm.integrations.websearch_interception.handler import (
WebSearchInterceptionLogger,
)
from litellm.types.integrations.custom_logger import (
RESPONSES_AGENTIC_SURFACE,
)
from litellm.llms.base_llm.search.transformation import SearchResponse
from litellm.proxy import proxy_server
from litellm.types.utils import CallTypes, LlmProviders
@ -252,6 +257,50 @@ async def test_build_responses_plan_produces_responses_input():
assert patch_obj.model == "openai/gpt-4o"
@pytest.mark.asyncio
async def test_build_responses_plan_keeps_the_provider_on_a_sub_path_model(monkeypatch: pytest.MonkeyPatch) -> None:
"""#38829 on the Responses surface: a provider-stripped model whose remainder still holds a
slash was left unqualified, so the follow-up could not resolve a provider."""
logger: Final = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.BEDROCK])
monkeypatch.setattr(proxy_server, "llm_router", None)
monkeypatch.setattr(litellm, "asearch", AsyncMock(return_value=SearchResponse(object="search", results=[])))
tools_dict: Final = {
"tool_calls": [
{
"id": "fc_1",
"call_id": "fc_1",
"type": "function_call",
"name": "litellm_web_search",
"arguments": '{"query": "latest ai news"}',
"input": {"query": "latest ai news"},
}
],
"tool_type": "websearch",
"provider": "bedrock",
"response_format": "responses",
}
plan: Final = await logger.async_build_responses_agentic_loop_plan(
tools=tools_dict,
model="mantle/anthropic.claude-sonnet-5",
messages=[{"role": "user", "content": "What's the latest AI news?"}],
response=_responses_output_with_web_search(),
optional_params={"tools": [{"type": "function", "name": "litellm_web_search"}]},
logging_obj=MagicMock(),
stream=False,
kwargs={
"custom_llm_provider": "bedrock",
"_agentic_loop_api_surface": RESPONSES_AGENTIC_SURFACE,
},
)
patch_obj: Final = plan.request_patch
assert patch_obj is not None
assert patch_obj.model == "bedrock/mantle/anthropic.claude-sonnet-5"
litellm.asearch.assert_awaited_once()
@pytest.mark.asyncio
async def test_deployment_hook_converts_native_responses_web_search_tool():
"""async_pre_call_deployment_hook converts a native Responses web_search tool

View file

@ -364,6 +364,49 @@ async def test_dispatcher_runs_followup_with_incremented_depth_and_patched_messa
assert logger.cleanup_calls == 1
@pytest.mark.asyncio
@pytest.mark.parametrize(
"model,custom_llm_provider,routed_model",
[
("cohere/command-a-03-2025", "cohere_chat", "command-a-03-2025"),
("anthropic/claude-2", "anthropic_text", "claude-2"),
],
)
async def test_sdk_followup_keeps_a_model_whose_prefix_already_routes_to_its_provider(
model: str, custom_llm_provider: str, routed_model: str
) -> None:
plan: Final = AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(messages=_patched_messages()),
)
gate: Final = _GateOnlyLogger(plan=plan, tool_calls={"tool_calls": [{"id": "call_abc"}]})
logging_obj: Final = Logging(
model=model,
messages=[{"role": "user", "content": "what is 6*7?"}],
stream=False,
call_type="acompletion",
start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
litellm_call_id="call-prefix",
function_id="fn-prefix",
dynamic_success_callbacks=[gate],
)
followup: Final = await maybe_run_chat_completion_agentic_loop(
response=_tool_call_model_response(),
model=model,
messages=[{"role": "user", "content": "what is 6*7?"}],
optional_params={},
kwargs={"mock_response": "done"},
logging_obj=logging_obj,
custom_llm_provider=custom_llm_provider,
stream=False,
)
assert isinstance(followup, ModelResponse)
assert followup.model == routed_model
assert followup._hidden_params["custom_llm_provider"] == custom_llm_provider
@pytest.mark.asyncio
async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_request_kwargs(
restore_callbacks,

View file

@ -26,6 +26,8 @@ from litellm.litellm_core_utils.core_helpers import (
normalize_drop_params,
preserve_upstream_non_openai_attributes,
process_response_headers,
qualify_agentic_followup_model,
qualify_provider_stripped_model,
reconstruct_model_name,
redact_nested_match_and_regex_keys,
set_provider_response_headers_in_hidden_params,
@ -636,6 +638,87 @@ class TestProviderResponseHeadersInHiddenParams:
assert "additional_headers" not in sibling._hidden_params
@pytest.mark.parametrize(
"model,provider,expected",
[
("mantle/anthropic.claude-sonnet-5", "bedrock", "bedrock/mantle/anthropic.claude-sonnet-5"),
("invoke/anthropic.claude-v2", "bedrock", "bedrock/invoke/anthropic.claude-v2"),
("openai/gpt-4o", "openrouter", "openrouter/openai/gpt-4o"),
("gpt-4o", "openai", "openai/gpt-4o"),
("claude-sonnet-4-5", "anthropic", "anthropic/claude-sonnet-4-5"),
],
)
def test_the_provider_prefix_is_restored(model: str, provider: str, expected: str) -> None:
assert qualify_provider_stripped_model(model, provider) == expected
@pytest.mark.parametrize(
"model,provider",
[
("bedrock/mantle/anthropic.claude-sonnet-5", "bedrock"),
("openai/gpt-4o", "openai"),
("cohere/command-a-03-2025", "cohere_chat"),
("ai21/jamba-1.5-large", "ai21_chat"),
("anthropic/claude-2", "anthropic_text"),
],
)
def test_an_already_qualified_model_is_left_alone(model: str, provider: str) -> None:
assert qualify_provider_stripped_model(model, provider) == model
def test_a_provider_that_only_shares_a_prefix_is_still_qualified() -> None:
assert qualify_provider_stripped_model("openai_like/foo", "openai") == "openai/openai_like/foo"
def test_no_provider_leaves_the_model_untouched() -> None:
assert qualify_provider_stripped_model("gpt-4o", "") == "gpt-4o"
def test_a_cross_provider_patched_model_is_dispatched_as_the_hook_asked() -> None:
assert qualify_agentic_followup_model("anthropic/claude-sonnet-4-5", "gpt-4o", "openai") == (
"anthropic/claude-sonnet-4-5"
)
def test_a_bare_patched_model_takes_the_request_provider() -> None:
assert qualify_agentic_followup_model("gpt-4o-mini", "gpt-4o", "openai") == "openai/gpt-4o-mini"
def test_a_sub_path_request_model_keeps_its_provider() -> None:
assert qualify_agentic_followup_model(None, "mantle/anthropic.claude-sonnet-5", "bedrock") == (
"bedrock/mantle/anthropic.claude-sonnet-5"
)
def test_an_unpatched_ordinary_model_takes_the_request_provider() -> None:
assert qualify_agentic_followup_model(None, "gpt-4o", "openai") == "openai/gpt-4o"
def test_a_patched_model_already_holding_the_request_provider_is_left_alone() -> None:
assert qualify_agentic_followup_model("openai/gpt-4o", "gpt-4o", "openai") == "openai/gpt-4o"
@pytest.mark.parametrize("patched", ["", None])
def test_a_patch_without_a_model_falls_back_to_the_request_model(patched: str | None) -> None:
assert qualify_agentic_followup_model(patched, "gpt-4o", "openai") == "openai/gpt-4o"
def test_a_patched_sub_path_model_still_takes_the_request_provider() -> None:
assert qualify_agentic_followup_model("mantle/anthropic.claude-sonnet-5", "gpt-4o", "bedrock") == (
"bedrock/mantle/anthropic.claude-sonnet-5"
)
def test_a_patched_model_on_a_json_configured_provider_reaches_that_provider() -> None:
assert qualify_agentic_followup_model("aihubmix/gpt-4o", "gpt-4o", "openai") == "aihubmix/gpt-4o"
def test_a_patch_echoing_the_request_model_keeps_the_request_provider() -> None:
assert qualify_agentic_followup_model("openai/gpt-4o", "openai/gpt-4o", "openrouter") == (
"openrouter/openai/gpt-4o"
)
def test_spend_metadata_merge_survives_the_logging_thread_inserting_into_the_request_metadata() -> None:
metadata: Final[dict[str, object]] = {}

View file

@ -5545,6 +5545,67 @@ async def test_anthropic_messages_dropped_safeguards_send_neither_the_field_nor_
assert "dangerous-tool-use-2026-09-03" not in sent.headers.get("anthropic-beta", ""), dict(sent.headers)
class TestAgenticFollowUpKeepsTheProviderPrefix:
"""#38829: the follow-up must re-dispatch a model litellm.acompletion can route."""
@staticmethod
def _plan():
from litellm.types.integrations.custom_logger import (
AgenticLoopPlan,
AgenticLoopRequestPatch,
)
return AgenticLoopPlan(
run_agentic_loop=True,
request_patch=AgenticLoopRequestPatch(messages=[{"role": "user", "content": "hi"}]),
)
async def _run_followup(self, model: str, custom_llm_provider: str, patched_model: str | None = None):
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
plan = self._plan()
if patched_model is not None:
plan.request_patch.model = patched_model
return await BaseLLMHTTPHandler()._execute_chat_completion_agentic_plan(
plan=plan,
model=model,
messages=[{"role": "user", "content": "hi"}],
optional_params={"mock_response": "ok from followup"},
kwargs={},
custom_llm_provider=custom_llm_provider,
depth=0,
max_loops=2,
fingerprints=[],
fingerprint="fp",
)
@pytest.mark.asyncio
async def test_a_sub_path_model_still_resolves_a_provider(self):
response = await self._run_followup("mantle/anthropic.claude-sonnet-5", "bedrock")
assert response.choices[0].message.content == "ok from followup"
@pytest.mark.asyncio
async def test_an_ordinary_model_still_resolves_a_provider(self):
response = await self._run_followup("gpt-4o-mini", "openai")
assert response.choices[0].message.content == "ok from followup"
@pytest.mark.asyncio
async def test_a_hook_that_patches_another_providers_model_reaches_that_provider(self):
response = await self._run_followup("gpt-4o", "openai", patched_model="anthropic/claude-sonnet-4-5")
assert response.model == "claude-sonnet-4-5"
assert response._hidden_params["custom_llm_provider"] == "anthropic"
@pytest.mark.asyncio
async def test_a_hook_echoing_an_org_style_model_stays_on_the_request_provider(self):
response = await self._run_followup("openai/gpt-4o", "openrouter", patched_model="openai/gpt-4o")
assert response._hidden_params["custom_llm_provider"] == "openrouter"
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("environment_base", [None, "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"])