mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 30d43f124a into e4832c5d06
This commit is contained in:
commit
f8ae64dcb4
8 changed files with 296 additions and 16 deletions
|
|
@ -37,6 +37,7 @@ from litellm.integrations.websearch_interception.transformation import (
|
|||
from litellm.litellm_core_utils.agentic_loop_settings import (
|
||||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import qualify_provider_stripped_model
|
||||
from litellm.llms.base_llm.search.transformation import SearchResponse
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
CHAT_COMPLETION_AGENTIC_SURFACE,
|
||||
|
|
@ -1251,9 +1252,10 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
}
|
||||
}
|
||||
|
||||
full_model_name = model
|
||||
if "/" not in model and isinstance(kwargs.get("custom_llm_provider"), str):
|
||||
full_model_name = f"{kwargs['custom_llm_provider']}/{model}"
|
||||
custom_llm_provider: Final = kwargs.get("custom_llm_provider")
|
||||
full_model_name: Final = qualify_provider_stripped_model(
|
||||
model, custom_llm_provider if isinstance(custom_llm_provider, str) else ""
|
||||
)
|
||||
|
||||
verbose_logger.debug(
|
||||
"WebSearchInterception: Built responses request patch model=%s input_items=%d searches=%d",
|
||||
|
|
@ -1833,7 +1835,7 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
messages: list[dict],
|
||||
tool_calls: list[dict],
|
||||
optional_params: dict,
|
||||
kwargs: dict,
|
||||
kwargs: dict[str, object],
|
||||
response_format: str = "openai",
|
||||
) -> AgenticLoopRequestPatch:
|
||||
"""Execute litellm.search() and build chat-completion rerun patch."""
|
||||
|
|
@ -1919,11 +1921,10 @@ class WebSearchInterceptionLogger(CustomLogger):
|
|||
k: v for k, v in kwargs.items() if not k.startswith("_websearch_interception") and k not in internal_params
|
||||
}
|
||||
|
||||
full_model_name = model
|
||||
if "custom_llm_provider" in kwargs:
|
||||
custom_llm_provider: Final = kwargs["custom_llm_provider"]
|
||||
if not model.startswith(custom_llm_provider) and "/" not in model:
|
||||
full_model_name = f"{custom_llm_provider}/{model}"
|
||||
custom_llm_provider: Final = kwargs.get("custom_llm_provider")
|
||||
full_model_name: Final = qualify_provider_stripped_model(
|
||||
model, custom_llm_provider if isinstance(custom_llm_provider, str) else ""
|
||||
)
|
||||
|
||||
verbose_logger.debug(
|
||||
"WebSearchInterception: Built chat completion request patch model=%s messages=%d",
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from litellm.litellm_core_utils.agentic_loop_settings import (
|
|||
DEFAULT_MAX_AGENTIC_LOOPS,
|
||||
validated_max_agentic_loops,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import qualify_agentic_followup_model
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObject
|
||||
from litellm.llms.base_llm.base_model_iterator import MockResponseIterator
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
|
|
@ -182,9 +183,7 @@ async def _execute_chat_completion_agentic_plan(
|
|||
if patch.messages is None:
|
||||
raise ValueError("Agentic loop plan missing patched messages")
|
||||
|
||||
full_model_name = patch.model or model
|
||||
if "/" not in full_model_name:
|
||||
full_model_name = f"{custom_llm_provider}/{full_model_name}"
|
||||
full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider)
|
||||
|
||||
optional_params_for_followup: Final = {**optional_params, **patch.optional_params}
|
||||
if patch.tools is not None:
|
||||
|
|
|
|||
|
|
@ -34,6 +34,48 @@ def is_codex_user_agent(user_agent: str) -> bool:
|
|||
return bool(_CODEX_CLIENT_PREFIX_RE.match(user_agent))
|
||||
|
||||
|
||||
def qualify_provider_stripped_model(model: str, custom_llm_provider: str) -> str:
|
||||
"""Put the provider prefix back on a provider-stripped model."""
|
||||
if not custom_llm_provider or model.startswith(f"{custom_llm_provider}/"):
|
||||
return model
|
||||
if _already_routes_to(model, custom_llm_provider):
|
||||
return model
|
||||
return f"{custom_llm_provider}/{model}"
|
||||
|
||||
|
||||
def _already_routes_to(model: str, custom_llm_provider: str) -> bool:
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import inferred_provider
|
||||
|
||||
leading_segment: Final = model.split("/", 1)[0]
|
||||
if leading_segment == model or not names_a_provider(leading_segment):
|
||||
return False
|
||||
return inferred_provider(model) == custom_llm_provider
|
||||
|
||||
|
||||
def names_a_provider(candidate: str) -> bool:
|
||||
"""Whether a model name's leading segment is a provider rather than part of the model."""
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
|
||||
provider_names: Final = {getattr(provider, "value", provider) for provider in litellm.provider_list}
|
||||
return (
|
||||
candidate in provider_names
|
||||
or JSONProviderRegistry.exists(candidate)
|
||||
or is_registered_custom_provider(candidate)
|
||||
)
|
||||
|
||||
|
||||
def qualify_agentic_followup_model(patch_model: str | None, model: str, custom_llm_provider: str) -> str:
|
||||
"""Resolve the model an agentic follow-up re-dispatches, keeping a hook's own provider."""
|
||||
if not patch_model or patch_model == model:
|
||||
return qualify_provider_stripped_model(model, custom_llm_provider)
|
||||
leading_segment: Final = patch_model.split("/", 1)[0]
|
||||
if leading_segment != patch_model and names_a_provider(leading_segment):
|
||||
return patch_model
|
||||
return qualify_provider_stripped_model(patch_model, custom_llm_provider)
|
||||
|
||||
|
||||
def safe_divide_seconds(seconds: float, denominator: float, default: float | None = None) -> float | None:
|
||||
"""
|
||||
Safely divide seconds by denominator, handling zero division.
|
||||
|
|
|
|||
|
|
@ -49,7 +49,11 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
|
|||
SUBTITLE_RESPONSE_FORMATS,
|
||||
synthesize_subtitle_document,
|
||||
)
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
as_str_mapping,
|
||||
qualify_agentic_followup_model,
|
||||
set_provider_response_headers_in_hidden_params,
|
||||
)
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
|
||||
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
|
||||
|
|
@ -6169,9 +6173,7 @@ class BaseLLMHTTPHandler:
|
|||
if patch.messages is None:
|
||||
raise ValueError("Agentic loop plan missing patched messages")
|
||||
|
||||
full_model_name = patch.model or model
|
||||
if "/" not in full_model_name:
|
||||
full_model_name = f"{custom_llm_provider}/{full_model_name}"
|
||||
full_model_name: Final = qualify_agentic_followup_model(patch.model, model, custom_llm_provider)
|
||||
|
||||
optional_params_for_followup: Final = dict(optional_params)
|
||||
optional_params_for_followup.update(patch.optional_params)
|
||||
|
|
|
|||
|
|
@ -7,16 +7,21 @@ builds a Responses-format follow-up request.
|
|||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
|
||||
from litellm.integrations.websearch_interception.handler import (
|
||||
WebSearchInterceptionLogger,
|
||||
)
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
RESPONSES_AGENTIC_SURFACE,
|
||||
)
|
||||
from litellm.llms.base_llm.search.transformation import SearchResponse
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.types.utils import CallTypes, LlmProviders
|
||||
|
||||
|
||||
|
|
@ -252,6 +257,50 @@ async def test_build_responses_plan_produces_responses_input():
|
|||
assert patch_obj.model == "openai/gpt-4o"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_build_responses_plan_keeps_the_provider_on_a_sub_path_model(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""#38829 on the Responses surface: a provider-stripped model whose remainder still holds a
|
||||
slash was left unqualified, so the follow-up could not resolve a provider."""
|
||||
logger: Final = WebSearchInterceptionLogger(enabled_providers=[LlmProviders.BEDROCK])
|
||||
monkeypatch.setattr(proxy_server, "llm_router", None)
|
||||
monkeypatch.setattr(litellm, "asearch", AsyncMock(return_value=SearchResponse(object="search", results=[])))
|
||||
|
||||
tools_dict: Final = {
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "fc_1",
|
||||
"call_id": "fc_1",
|
||||
"type": "function_call",
|
||||
"name": "litellm_web_search",
|
||||
"arguments": '{"query": "latest ai news"}',
|
||||
"input": {"query": "latest ai news"},
|
||||
}
|
||||
],
|
||||
"tool_type": "websearch",
|
||||
"provider": "bedrock",
|
||||
"response_format": "responses",
|
||||
}
|
||||
|
||||
plan: Final = await logger.async_build_responses_agentic_loop_plan(
|
||||
tools=tools_dict,
|
||||
model="mantle/anthropic.claude-sonnet-5",
|
||||
messages=[{"role": "user", "content": "What's the latest AI news?"}],
|
||||
response=_responses_output_with_web_search(),
|
||||
optional_params={"tools": [{"type": "function", "name": "litellm_web_search"}]},
|
||||
logging_obj=MagicMock(),
|
||||
stream=False,
|
||||
kwargs={
|
||||
"custom_llm_provider": "bedrock",
|
||||
"_agentic_loop_api_surface": RESPONSES_AGENTIC_SURFACE,
|
||||
},
|
||||
)
|
||||
|
||||
patch_obj: Final = plan.request_patch
|
||||
assert patch_obj is not None
|
||||
assert patch_obj.model == "bedrock/mantle/anthropic.claude-sonnet-5"
|
||||
litellm.asearch.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_deployment_hook_converts_native_responses_web_search_tool():
|
||||
"""async_pre_call_deployment_hook converts a native Responses web_search tool
|
||||
|
|
|
|||
|
|
@ -364,6 +364,49 @@ async def test_dispatcher_runs_followup_with_incremented_depth_and_patched_messa
|
|||
assert logger.cleanup_calls == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"model,custom_llm_provider,routed_model",
|
||||
[
|
||||
("cohere/command-a-03-2025", "cohere_chat", "command-a-03-2025"),
|
||||
("anthropic/claude-2", "anthropic_text", "claude-2"),
|
||||
],
|
||||
)
|
||||
async def test_sdk_followup_keeps_a_model_whose_prefix_already_routes_to_its_provider(
|
||||
model: str, custom_llm_provider: str, routed_model: str
|
||||
) -> None:
|
||||
plan: Final = AgenticLoopPlan(
|
||||
run_agentic_loop=True,
|
||||
request_patch=AgenticLoopRequestPatch(messages=_patched_messages()),
|
||||
)
|
||||
gate: Final = _GateOnlyLogger(plan=plan, tool_calls={"tool_calls": [{"id": "call_abc"}]})
|
||||
logging_obj: Final = Logging(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "what is 6*7?"}],
|
||||
stream=False,
|
||||
call_type="acompletion",
|
||||
start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc),
|
||||
litellm_call_id="call-prefix",
|
||||
function_id="fn-prefix",
|
||||
dynamic_success_callbacks=[gate],
|
||||
)
|
||||
|
||||
followup: Final = await maybe_run_chat_completion_agentic_loop(
|
||||
response=_tool_call_model_response(),
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "what is 6*7?"}],
|
||||
optional_params={},
|
||||
kwargs={"mock_response": "done"},
|
||||
logging_obj=logging_obj,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
stream=False,
|
||||
)
|
||||
|
||||
assert isinstance(followup, ModelResponse)
|
||||
assert followup.model == routed_model
|
||||
assert followup._hidden_params["custom_llm_provider"] == custom_llm_provider
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dispatcher_followup_does_not_repeat_a_request_param_found_in_request_kwargs(
|
||||
restore_callbacks,
|
||||
|
|
|
|||
|
|
@ -26,6 +26,8 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
normalize_drop_params,
|
||||
preserve_upstream_non_openai_attributes,
|
||||
process_response_headers,
|
||||
qualify_agentic_followup_model,
|
||||
qualify_provider_stripped_model,
|
||||
reconstruct_model_name,
|
||||
redact_nested_match_and_regex_keys,
|
||||
set_provider_response_headers_in_hidden_params,
|
||||
|
|
@ -636,6 +638,87 @@ class TestProviderResponseHeadersInHiddenParams:
|
|||
assert "additional_headers" not in sibling._hidden_params
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,provider,expected",
|
||||
[
|
||||
("mantle/anthropic.claude-sonnet-5", "bedrock", "bedrock/mantle/anthropic.claude-sonnet-5"),
|
||||
("invoke/anthropic.claude-v2", "bedrock", "bedrock/invoke/anthropic.claude-v2"),
|
||||
("openai/gpt-4o", "openrouter", "openrouter/openai/gpt-4o"),
|
||||
("gpt-4o", "openai", "openai/gpt-4o"),
|
||||
("claude-sonnet-4-5", "anthropic", "anthropic/claude-sonnet-4-5"),
|
||||
],
|
||||
)
|
||||
def test_the_provider_prefix_is_restored(model: str, provider: str, expected: str) -> None:
|
||||
assert qualify_provider_stripped_model(model, provider) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"model,provider",
|
||||
[
|
||||
("bedrock/mantle/anthropic.claude-sonnet-5", "bedrock"),
|
||||
("openai/gpt-4o", "openai"),
|
||||
("cohere/command-a-03-2025", "cohere_chat"),
|
||||
("ai21/jamba-1.5-large", "ai21_chat"),
|
||||
("anthropic/claude-2", "anthropic_text"),
|
||||
],
|
||||
)
|
||||
def test_an_already_qualified_model_is_left_alone(model: str, provider: str) -> None:
|
||||
assert qualify_provider_stripped_model(model, provider) == model
|
||||
|
||||
|
||||
def test_a_provider_that_only_shares_a_prefix_is_still_qualified() -> None:
|
||||
assert qualify_provider_stripped_model("openai_like/foo", "openai") == "openai/openai_like/foo"
|
||||
|
||||
|
||||
def test_no_provider_leaves_the_model_untouched() -> None:
|
||||
assert qualify_provider_stripped_model("gpt-4o", "") == "gpt-4o"
|
||||
|
||||
|
||||
def test_a_cross_provider_patched_model_is_dispatched_as_the_hook_asked() -> None:
|
||||
assert qualify_agentic_followup_model("anthropic/claude-sonnet-4-5", "gpt-4o", "openai") == (
|
||||
"anthropic/claude-sonnet-4-5"
|
||||
)
|
||||
|
||||
|
||||
def test_a_bare_patched_model_takes_the_request_provider() -> None:
|
||||
assert qualify_agentic_followup_model("gpt-4o-mini", "gpt-4o", "openai") == "openai/gpt-4o-mini"
|
||||
|
||||
|
||||
def test_a_sub_path_request_model_keeps_its_provider() -> None:
|
||||
assert qualify_agentic_followup_model(None, "mantle/anthropic.claude-sonnet-5", "bedrock") == (
|
||||
"bedrock/mantle/anthropic.claude-sonnet-5"
|
||||
)
|
||||
|
||||
|
||||
def test_an_unpatched_ordinary_model_takes_the_request_provider() -> None:
|
||||
assert qualify_agentic_followup_model(None, "gpt-4o", "openai") == "openai/gpt-4o"
|
||||
|
||||
|
||||
def test_a_patched_model_already_holding_the_request_provider_is_left_alone() -> None:
|
||||
assert qualify_agentic_followup_model("openai/gpt-4o", "gpt-4o", "openai") == "openai/gpt-4o"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("patched", ["", None])
|
||||
def test_a_patch_without_a_model_falls_back_to_the_request_model(patched: str | None) -> None:
|
||||
assert qualify_agentic_followup_model(patched, "gpt-4o", "openai") == "openai/gpt-4o"
|
||||
|
||||
|
||||
def test_a_patched_sub_path_model_still_takes_the_request_provider() -> None:
|
||||
assert qualify_agentic_followup_model("mantle/anthropic.claude-sonnet-5", "gpt-4o", "bedrock") == (
|
||||
"bedrock/mantle/anthropic.claude-sonnet-5"
|
||||
)
|
||||
|
||||
|
||||
def test_a_patched_model_on_a_json_configured_provider_reaches_that_provider() -> None:
|
||||
assert qualify_agentic_followup_model("aihubmix/gpt-4o", "gpt-4o", "openai") == "aihubmix/gpt-4o"
|
||||
|
||||
|
||||
def test_a_patch_echoing_the_request_model_keeps_the_request_provider() -> None:
|
||||
assert qualify_agentic_followup_model("openai/gpt-4o", "openai/gpt-4o", "openrouter") == (
|
||||
"openrouter/openai/gpt-4o"
|
||||
)
|
||||
|
||||
|
||||
def test_spend_metadata_merge_survives_the_logging_thread_inserting_into_the_request_metadata() -> None:
|
||||
metadata: Final[dict[str, object]] = {}
|
||||
|
||||
|
|
|
|||
|
|
@ -5545,6 +5545,67 @@ async def test_anthropic_messages_dropped_safeguards_send_neither_the_field_nor_
|
|||
assert "dangerous-tool-use-2026-09-03" not in sent.headers.get("anthropic-beta", ""), dict(sent.headers)
|
||||
|
||||
|
||||
class TestAgenticFollowUpKeepsTheProviderPrefix:
|
||||
"""#38829: the follow-up must re-dispatch a model litellm.acompletion can route."""
|
||||
|
||||
@staticmethod
|
||||
def _plan():
|
||||
from litellm.types.integrations.custom_logger import (
|
||||
AgenticLoopPlan,
|
||||
AgenticLoopRequestPatch,
|
||||
)
|
||||
|
||||
return AgenticLoopPlan(
|
||||
run_agentic_loop=True,
|
||||
request_patch=AgenticLoopRequestPatch(messages=[{"role": "user", "content": "hi"}]),
|
||||
)
|
||||
|
||||
async def _run_followup(self, model: str, custom_llm_provider: str, patched_model: str | None = None):
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
|
||||
plan = self._plan()
|
||||
if patched_model is not None:
|
||||
plan.request_patch.model = patched_model
|
||||
|
||||
return await BaseLLMHTTPHandler()._execute_chat_completion_agentic_plan(
|
||||
plan=plan,
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
optional_params={"mock_response": "ok from followup"},
|
||||
kwargs={},
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
depth=0,
|
||||
max_loops=2,
|
||||
fingerprints=[],
|
||||
fingerprint="fp",
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_sub_path_model_still_resolves_a_provider(self):
|
||||
response = await self._run_followup("mantle/anthropic.claude-sonnet-5", "bedrock")
|
||||
|
||||
assert response.choices[0].message.content == "ok from followup"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_ordinary_model_still_resolves_a_provider(self):
|
||||
response = await self._run_followup("gpt-4o-mini", "openai")
|
||||
|
||||
assert response.choices[0].message.content == "ok from followup"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_hook_that_patches_another_providers_model_reaches_that_provider(self):
|
||||
response = await self._run_followup("gpt-4o", "openai", patched_model="anthropic/claude-sonnet-4-5")
|
||||
|
||||
assert response.model == "claude-sonnet-4-5"
|
||||
assert response._hidden_params["custom_llm_provider"] == "anthropic"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_hook_echoing_an_org_style_model_stays_on_the_request_provider(self):
|
||||
response = await self._run_followup("openai/gpt-4o", "openrouter", patched_model="openai/gpt-4o")
|
||||
|
||||
assert response._hidden_params["custom_llm_provider"] == "openrouter"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize("environment_base", [None, "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"])
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue