scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 36

fix(advisor): resolve the advisor sub-call through the proxy router

Browse source 
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
shivam 2026-08-08 02:20:35 +00:00
parent ecb0ea2f1c
commit 3c96030488
2 changed files with 264 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py
View file

@ -16,7 +16,7 @@ How it works:
import uuid
from collections.abc import AsyncIterator
from typing import Any, Final
from typing import TYPE_CHECKING, Any, Final, cast
import litellm
import litellm.constants as _c
@ -28,6 +28,9 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
AnthropicMessagesResponse,
)
if TYPE_CHECKING:
from litellm.router import Router
ADVISOR_MAX_USES: Final[int] = _c.ADVISOR_MAX_USES
ADVISOR_NATIVE_PROVIDERS: Final[frozenset] = _c.ADVISOR_NATIVE_PROVIDERS
ADVISOR_TOOL_DESCRIPTION: Final[str] = _c.ADVISOR_TOOL_DESCRIPTION
@ -138,13 +141,10 @@ class AdvisorOrchestrationHandler(MessagesInterceptor):
# --- Advisor sub-call (always non-streaming, no tools) ---
try:
advisor_response: AnthropicMessagesResponse = await _call_messages_handler(
advisor_response: AnthropicMessagesResponse = await _call_advisor(
model=advisor_model,
messages=advisor_messages,
tools=None,
stream=False,
max_tokens=max_tokens,
custom_llm_provider=None, # let litellm resolve from model name
metadata={
**metadata_base,
"advisor_sub_call": True,
@ -357,6 +357,75 @@ def _inject_max_uses_error(
]
def _resolve_advisor_router(advisor_model: str) -> "Router | None":
"""Return the proxy router when it can resolve ``advisor_model``.
The advisor sub-call must honor the proxy's ``model_list`` (and its
fallbacks / credentials) exactly like a direct call to that model group
would. Without this, provider resolution falls back to the bare model
name, which for a ``claude-*`` advisor model means the public Anthropic
API, bypassing the configured deployment entirely.
Returns ``None`` for SDK callers (no proxy router) and for advisor models
the router doesn't know about, so those keep resolving through
``litellm.anthropic_messages()`` provider inference.
"""
try:
from litellm.proxy.proxy_server import llm_router
except (ImportError, ModuleNotFoundError):
return None
if llm_router is None:
return None
if llm_router.get_model_list(model_name=advisor_model):
return llm_router
if llm_router.model_group_alias and advisor_model in llm_router.model_group_alias:
return llm_router
if llm_router.pattern_router.route(advisor_model) is not None:
return llm_router
return None
async def _call_advisor(
*,
model: str,
messages: list[dict],
max_tokens: int,
metadata: dict,
api_key: str | None,
api_base: str | None,
) -> AnthropicMessagesResponse:
"""Run the advisor sub-call, through the proxy router when it applies.
A caller-supplied ``api_key`` / ``api_base`` override is an explicit
request to bypass the configured deployment, so it keeps the direct
SDK-level path.
"""
router: Final = None if (api_key or api_base) else _resolve_advisor_router(model)
response: Final = (
await router.aanthropic_messages(
model=model,
messages=messages,
tools=None,
stream=False,
max_tokens=max_tokens,
metadata=metadata,
)
if router is not None
else await _call_messages_handler(
model=model,
messages=messages,
tools=None,
stream=False,
max_tokens=max_tokens,
custom_llm_provider=None,
metadata=metadata,
api_key=api_key,
api_base=api_base,
)
)
return cast(AnthropicMessagesResponse, response) # cast-ok: both /messages entry points are untyped
async def _call_messages_handler(
model: str,
messages: list[dict],

190
tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py
View file

@ -1041,3 +1041,193 @@ async def test_executor_failure_is_not_tagged():
)
assert is_advisor_orchestration_failure(exc_info.value) is False
# ---------------------------------------------------------------------------
# 15. The advisor sub-call resolves through the proxy router when the advisor
# model is configured in model_list, instead of dialing the public
# Anthropic API (regression for LIT-5307).
# ---------------------------------------------------------------------------
def _router_with_advisor_deployment(recorder, advisor_model="claude-opus-4-8"):
"""Build a Router whose only deployment is the advisor model on Foundry.
The recorder replaces ``litellm.anthropic_messages`` before construction
because Router binds it at init time, so the returned Router exercises the
real deployment-resolution path and records what it dispatched.
"""
import litellm
from litellm.router import Router
with patch("litellm.anthropic_messages", new=recorder):
return Router(
model_list=[
{
"model_name": advisor_model,
"litellm_params": {
"model": f"azure_ai/{advisor_model}",
"api_base": "http://127.0.0.1:1/foundry",
"api_key": "fake-foundry-key",
},
}
],
num_retries=0,
)
@pytest.mark.asyncio
async def test_advisor_sub_call_routes_through_proxy_router():
import litellm.proxy.proxy_server as proxy_server
from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import (
AdvisorOrchestrationHandler,
)
router_calls = []
async def recorder(**kwargs):
router_calls.append(kwargs)
return _make_text_response("Use trial division.", model="claude-opus-4-8")
router = _router_with_advisor_deployment(recorder)
call_count = 0
async def mock_call(model, messages, tools, stream, max_tokens, **kwargs):
nonlocal call_count
call_count += 1
if call_count == 1:
return _make_advisor_tool_use_response()
return _make_text_response("Final answer.")
with (
patch(
"litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler",
side_effect=mock_call,
),
patch.object(proxy_server, "llm_router", router),
):
h = AdvisorOrchestrationHandler()
result = await h.handle(
model="executor-model",
messages=MESSAGES,
tools=[{**ADVISOR_TOOL, "model": "claude-opus-4-8"}],
stream=False,
max_tokens=512,
custom_llm_provider="azure_ai",
)
assert call_count == 2
assert len(router_calls) == 1
assert router_calls[0]["model"] == "azure_ai/claude-opus-4-8"
assert router_calls[0]["api_base"] == "http://127.0.0.1:1/foundry"
assert router_calls[0]["api_key"] == "fake-foundry-key"
assert "Final answer." in result["content"][0]["text"]
@pytest.mark.asyncio
async def test_advisor_sub_call_bypasses_router_for_unconfigured_model():
"""An advisor model the router doesn't know about keeps the SDK-level path."""
import litellm.proxy.proxy_server as proxy_server
from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import (
AdvisorOrchestrationHandler,
)
router_calls = []
async def recorder(**kwargs):
router_calls.append(kwargs)
return _make_text_response("should not be used")
router = _router_with_advisor_deployment(recorder, advisor_model="some-other-model")
call_count = 0
async def mock_call(model, messages, tools, stream, max_tokens, **kwargs):
nonlocal call_count
call_count += 1
if call_count == 1:
return _make_advisor_tool_use_response()
if tools is None:
return _make_text_response("Advice.", model="claude-opus-4-8")
return _make_text_response("Final answer.")
with (
patch(
"litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler",
side_effect=mock_call,
),
patch.object(proxy_server, "llm_router", router),
):
h = AdvisorOrchestrationHandler()
await h.handle(
model="executor-model",
messages=MESSAGES,
tools=[{**ADVISOR_TOOL, "model": "claude-opus-4-8"}],
stream=False,
max_tokens=512,
custom_llm_provider="azure_ai",
)
assert router_calls == []
assert call_count == 3
@pytest.mark.asyncio
async def test_advisor_sub_call_client_override_bypasses_router():
"""A caller-supplied api_key/api_base override must not be re-routed."""
import litellm
import litellm.proxy.proxy_server as proxy_server
from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import (
AdvisorOrchestrationHandler,
)
router_calls = []
async def recorder(**kwargs):
router_calls.append(kwargs)
return _make_text_response("should not be used")
router = _router_with_advisor_deployment(recorder)
sub_calls = []
async def mock_call(model, messages, tools, stream, max_tokens, **kwargs):
sub_calls.append({"model": model, "tools": tools, **kwargs})
if len(sub_calls) == 1:
return _make_advisor_tool_use_response()
if tools is None:
return _make_text_response("Advice.", model="claude-opus-4-8")
return _make_text_response("Final answer.")
advisor_tool = {
**ADVISOR_TOOL,
"model": "claude-opus-4-8",
"api_key": "client-key",
"api_base": "https://client.example.com",
}
with (
patch(
"litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler",
side_effect=mock_call,
),
patch.object(proxy_server, "llm_router", router),
patch.dict(proxy_server.general_settings, {"allow_client_side_credentials": True}),
patch.object(litellm, "user_url_validation", False),
):
h = AdvisorOrchestrationHandler()
await h.handle(
model="executor-model",
messages=MESSAGES,
tools=[advisor_tool],
stream=False,
max_tokens=512,
custom_llm_provider="azure_ai",
)
assert router_calls == []
advisor_sub_calls = [c for c in sub_calls if c["tools"] is None]
assert len(advisor_sub_calls) == 1
assert advisor_sub_calls[0]["api_key"] == "client-key"
assert advisor_sub_calls[0]["api_base"] == "https://client.example.com"