diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py index dfae7b4f4cf..805f5625ad9 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py @@ -16,7 +16,7 @@ How it works: import uuid from collections.abc import AsyncIterator -from typing import Any, Final +from typing import TYPE_CHECKING, Any, Final, cast import litellm import litellm.constants as _c @@ -28,6 +28,9 @@ from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) +if TYPE_CHECKING: + from litellm.router import Router + ADVISOR_MAX_USES: Final[int] = _c.ADVISOR_MAX_USES ADVISOR_NATIVE_PROVIDERS: Final[frozenset] = _c.ADVISOR_NATIVE_PROVIDERS ADVISOR_TOOL_DESCRIPTION: Final[str] = _c.ADVISOR_TOOL_DESCRIPTION @@ -138,13 +141,10 @@ class AdvisorOrchestrationHandler(MessagesInterceptor): # --- Advisor sub-call (always non-streaming, no tools) --- try: - advisor_response: AnthropicMessagesResponse = await _call_messages_handler( + advisor_response: AnthropicMessagesResponse = await _call_advisor( model=advisor_model, messages=advisor_messages, - tools=None, - stream=False, max_tokens=max_tokens, - custom_llm_provider=None, # let litellm resolve from model name metadata={ **metadata_base, "advisor_sub_call": True, @@ -357,6 +357,75 @@ def _inject_max_uses_error( ] +def _resolve_advisor_router(advisor_model: str) -> "Router | None": + """Return the proxy router when it can resolve ``advisor_model``. + + The advisor sub-call must honor the proxy's ``model_list`` (and its + fallbacks / credentials) exactly like a direct call to that model group + would. Without this, provider resolution falls back to the bare model + name, which for a ``claude-*`` advisor model means the public Anthropic + API, bypassing the configured deployment entirely. + + Returns ``None`` for SDK callers (no proxy router) and for advisor models + the router doesn't know about, so those keep resolving through + ``litellm.anthropic_messages()`` provider inference. + """ + try: + from litellm.proxy.proxy_server import llm_router + except (ImportError, ModuleNotFoundError): + return None + if llm_router is None: + return None + if llm_router.get_model_list(model_name=advisor_model): + return llm_router + if llm_router.model_group_alias and advisor_model in llm_router.model_group_alias: + return llm_router + if llm_router.pattern_router.route(advisor_model) is not None: + return llm_router + return None + + +async def _call_advisor( + *, + model: str, + messages: list[dict], + max_tokens: int, + metadata: dict, + api_key: str | None, + api_base: str | None, +) -> AnthropicMessagesResponse: + """Run the advisor sub-call, through the proxy router when it applies. + + A caller-supplied ``api_key`` / ``api_base`` override is an explicit + request to bypass the configured deployment, so it keeps the direct + SDK-level path. + """ + router: Final = None if (api_key or api_base) else _resolve_advisor_router(model) + response: Final = ( + await router.aanthropic_messages( + model=model, + messages=messages, + tools=None, + stream=False, + max_tokens=max_tokens, + metadata=metadata, + ) + if router is not None + else await _call_messages_handler( + model=model, + messages=messages, + tools=None, + stream=False, + max_tokens=max_tokens, + custom_llm_provider=None, + metadata=metadata, + api_key=api_key, + api_base=api_base, + ) + ) + return cast(AnthropicMessagesResponse, response) # cast-ok: both /messages entry points are untyped + + async def _call_messages_handler( model: str, messages: list[dict], diff --git a/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py b/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py index 3d35e93167f..bcb1843f408 100644 --- a/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py +++ b/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py @@ -1041,3 +1041,193 @@ async def test_executor_failure_is_not_tagged(): ) assert is_advisor_orchestration_failure(exc_info.value) is False + + +# --------------------------------------------------------------------------- +# 15. The advisor sub-call resolves through the proxy router when the advisor +# model is configured in model_list, instead of dialing the public +# Anthropic API (regression for LIT-5307). +# --------------------------------------------------------------------------- + + +def _router_with_advisor_deployment(recorder, advisor_model="claude-opus-4-8"): + """Build a Router whose only deployment is the advisor model on Foundry. + + The recorder replaces ``litellm.anthropic_messages`` before construction + because Router binds it at init time, so the returned Router exercises the + real deployment-resolution path and records what it dispatched. + """ + import litellm + from litellm.router import Router + + with patch("litellm.anthropic_messages", new=recorder): + return Router( + model_list=[ + { + "model_name": advisor_model, + "litellm_params": { + "model": f"azure_ai/{advisor_model}", + "api_base": "http://127.0.0.1:1/foundry", + "api_key": "fake-foundry-key", + }, + } + ], + num_retries=0, + ) + + +@pytest.mark.asyncio +async def test_advisor_sub_call_routes_through_proxy_router(): + import litellm.proxy.proxy_server as proxy_server + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import ( + AdvisorOrchestrationHandler, + ) + + router_calls = [] + + async def recorder(**kwargs): + router_calls.append(kwargs) + return _make_text_response("Use trial division.", model="claude-opus-4-8") + + router = _router_with_advisor_deployment(recorder) + + call_count = 0 + + async def mock_call(model, messages, tools, stream, max_tokens, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + return _make_advisor_tool_use_response() + return _make_text_response("Final answer.") + + with ( + patch( + "litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler", + side_effect=mock_call, + ), + patch.object(proxy_server, "llm_router", router), + ): + h = AdvisorOrchestrationHandler() + result = await h.handle( + model="executor-model", + messages=MESSAGES, + tools=[{**ADVISOR_TOOL, "model": "claude-opus-4-8"}], + stream=False, + max_tokens=512, + custom_llm_provider="azure_ai", + ) + + assert call_count == 2 + assert len(router_calls) == 1 + assert router_calls[0]["model"] == "azure_ai/claude-opus-4-8" + assert router_calls[0]["api_base"] == "http://127.0.0.1:1/foundry" + assert router_calls[0]["api_key"] == "fake-foundry-key" + assert "Final answer." in result["content"][0]["text"] + + +@pytest.mark.asyncio +async def test_advisor_sub_call_bypasses_router_for_unconfigured_model(): + """An advisor model the router doesn't know about keeps the SDK-level path.""" + import litellm.proxy.proxy_server as proxy_server + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import ( + AdvisorOrchestrationHandler, + ) + + router_calls = [] + + async def recorder(**kwargs): + router_calls.append(kwargs) + return _make_text_response("should not be used") + + router = _router_with_advisor_deployment(recorder, advisor_model="some-other-model") + + call_count = 0 + + async def mock_call(model, messages, tools, stream, max_tokens, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + return _make_advisor_tool_use_response() + if tools is None: + return _make_text_response("Advice.", model="claude-opus-4-8") + return _make_text_response("Final answer.") + + with ( + patch( + "litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler", + side_effect=mock_call, + ), + patch.object(proxy_server, "llm_router", router), + ): + h = AdvisorOrchestrationHandler() + await h.handle( + model="executor-model", + messages=MESSAGES, + tools=[{**ADVISOR_TOOL, "model": "claude-opus-4-8"}], + stream=False, + max_tokens=512, + custom_llm_provider="azure_ai", + ) + + assert router_calls == [] + assert call_count == 3 + + +@pytest.mark.asyncio +async def test_advisor_sub_call_client_override_bypasses_router(): + """A caller-supplied api_key/api_base override must not be re-routed.""" + import litellm + import litellm.proxy.proxy_server as proxy_server + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import ( + AdvisorOrchestrationHandler, + ) + + router_calls = [] + + async def recorder(**kwargs): + router_calls.append(kwargs) + return _make_text_response("should not be used") + + router = _router_with_advisor_deployment(recorder) + + sub_calls = [] + + async def mock_call(model, messages, tools, stream, max_tokens, **kwargs): + sub_calls.append({"model": model, "tools": tools, **kwargs}) + if len(sub_calls) == 1: + return _make_advisor_tool_use_response() + if tools is None: + return _make_text_response("Advice.", model="claude-opus-4-8") + return _make_text_response("Final answer.") + + advisor_tool = { + **ADVISOR_TOOL, + "model": "claude-opus-4-8", + "api_key": "client-key", + "api_base": "https://client.example.com", + } + + with ( + patch( + "litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler", + side_effect=mock_call, + ), + patch.object(proxy_server, "llm_router", router), + patch.dict(proxy_server.general_settings, {"allow_client_side_credentials": True}), + patch.object(litellm, "user_url_validation", False), + ): + h = AdvisorOrchestrationHandler() + await h.handle( + model="executor-model", + messages=MESSAGES, + tools=[advisor_tool], + stream=False, + max_tokens=512, + custom_llm_provider="azure_ai", + ) + + assert router_calls == [] + advisor_sub_calls = [c for c in sub_calls if c["tools"] is None] + assert len(advisor_sub_calls) == 1 + assert advisor_sub_calls[0]["api_key"] == "client-key" + assert advisor_sub_calls[0]["api_base"] == "https://client.example.com"