diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py index 79faa39c7a2..a36f825951a 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/interceptors/advisor.py @@ -21,6 +21,7 @@ import litellm import litellm.constants as _c from litellm.litellm_core_utils.url_utils import validate_url from litellm.llms.anthropic.common_utils import strip_advisor_blocks_from_messages +from litellm.router_utils.cooldown_handlers import mark_advisor_orchestration_failure from litellm.types.llms.anthropic_messages.anthropic_response import ( AnthropicMessagesResponse, ) @@ -124,30 +125,36 @@ class AdvisorOrchestrationHandler(MessagesInterceptor): iteration += 1 if iteration > max_uses: - raise AdvisorMaxIterationsError( + max_iterations_error = AdvisorMaxIterationsError( f"Advisor orchestration loop exceeded max_uses={max_uses}. " "Increase max_uses in the advisor tool definition or cap the request." ) + mark_advisor_orchestration_failure(max_iterations_error) + raise max_iterations_error # --- Build advisor context --- advisor_messages = _build_advisor_context(current_messages, executor_response, advisor_use_block) # --- Advisor sub-call (always non-streaming, no tools) --- - advisor_response: AnthropicMessagesResponse = await _call_messages_handler( - model=advisor_model, - messages=advisor_messages, - tools=None, - stream=False, - max_tokens=max_tokens, - custom_llm_provider=None, # let litellm resolve from model name - metadata={ - **metadata_base, - "advisor_sub_call": True, - "parent_request_id": parent_request_id, - }, - api_key=advisor_api_key, - api_base=advisor_api_base, - ) + try: + advisor_response: AnthropicMessagesResponse = await _call_messages_handler( + model=advisor_model, + messages=advisor_messages, + tools=None, + stream=False, + max_tokens=max_tokens, + custom_llm_provider=None, # let litellm resolve from model name + metadata={ + **metadata_base, + "advisor_sub_call": True, + "parent_request_id": parent_request_id, + }, + api_key=advisor_api_key, + api_base=advisor_api_base, + ) + except Exception as advisor_sub_call_exception: + mark_advisor_orchestration_failure(advisor_sub_call_exception) + raise advisor_text = _extract_response_text(advisor_response) diff --git a/litellm/router.py b/litellm/router.py index 3ecaef591f3..78fe3ff025e 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -126,6 +126,7 @@ from litellm.router_utils.cooldown_handlers import ( _async_get_cooldown_deployments_with_debug_info, _get_cooldown_deployments, _set_cooldown_deployments, + is_advisor_orchestration_failure, ) from litellm.router_utils.fallback_event_handlers import ( _check_non_standard_fallback_format, @@ -7005,6 +7006,14 @@ class Router: verbose_router_logger.debug("Router: Entering 'deployment_callback_on_failure'") try: exception = kwargs.get("exception", None) + + if is_advisor_orchestration_failure(exception): + verbose_router_logger.debug( + "Router: Exiting 'deployment_callback_on_failure' without cooldown. " + "Failure originated from advisor orchestration, not the selected deployment." + ) + return False + exception_status = getattr(exception, "status_code", "") # Cache litellm_params to avoid repeated dict lookups diff --git a/litellm/router_utils/cooldown_handlers.py b/litellm/router_utils/cooldown_handlers.py index 2bc2ed998ca..c1fc939880a 100644 --- a/litellm/router_utils/cooldown_handlers.py +++ b/litellm/router_utils/cooldown_handlers.py @@ -36,6 +36,27 @@ else: LitellmRouter = Any Span = Any +_ADVISOR_ORCHESTRATION_FAILURE_ATTR = "_litellm_advisor_orchestration_failure" + + +def mark_advisor_orchestration_failure(exception: BaseException) -> None: + """Tag an exception as originating from advisor orchestration rather than the + health of the router-selected deployment. + + Advisor orchestration failures (an advisor sub-call that targets different + provider/credentials, or the orchestration loop exceeding max_uses) are not + caused by the selected deployment, so they must not be attributed to (and + cool down) that otherwise-healthy deployment. The exception object is tagged + rather than wrapped so its type is preserved and the router's retry/fallback + classification and the client-facing error are unchanged. + """ + setattr(exception, _ADVISOR_ORCHESTRATION_FAILURE_ATTR, True) + + +def is_advisor_orchestration_failure(exception: BaseException | None) -> bool: + """Whether ``exception`` was tagged by ``mark_advisor_orchestration_failure``.""" + return bool(getattr(exception, _ADVISOR_ORCHESTRATION_FAILURE_ATTR, False)) + def _is_cooldown_required( litellm_router_instance: LitellmRouter, diff --git a/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py b/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py index a2f5e00c8aa..3d35e93167f 100644 --- a/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py +++ b/tests/test_litellm/llms/anthropic/messages/test_advisor_orchestration.py @@ -919,3 +919,125 @@ def test_resolve_advisor_credentials_allows_real_public_ip_address(): ): result = _resolve_advisor_credentials(tool) assert result == ("sk-other", "https://8.8.8.8") + + +# --------------------------------------------------------------------------- +# 14. Advisor orchestration failures (a sub-call failure or the loop exceeding +# max_uses) are tagged so the router does not cool down the (healthy) parent +# deployment; executor failures are NOT tagged (regression for LIT-4565). +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_advisor_sub_call_failure_is_tagged(): + """When the advisor sub-call raises, the exception that propagates out of + handle() must be tagged as an advisor orchestration failure.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import ( + AdvisorOrchestrationHandler, + ) + from litellm.router_utils.cooldown_handlers import is_advisor_orchestration_failure + + call_count = 0 + + async def mock_call(model, messages, tools, stream, max_tokens, **kwargs): + nonlocal call_count + call_count += 1 + if call_count == 1: + return _make_advisor_tool_use_response() # executor: calls advisor + raise litellm.AuthenticationError( # advisor sub-call: 401 + message="x-api-key header is required", + llm_provider="anthropic", + model=model, + ) + + with patch( + "litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler", + side_effect=mock_call, + ): + h = AdvisorOrchestrationHandler() + with pytest.raises(litellm.AuthenticationError) as exc_info: + await h.handle( + model="openai/gpt-4o-mini", + messages=MESSAGES, + tools=[ADVISOR_TOOL], + stream=False, + max_tokens=512, + custom_llm_provider="openai", + ) + + assert call_count == 2 + assert is_advisor_orchestration_failure(exc_info.value) is True + + +@pytest.mark.asyncio +async def test_advisor_max_iterations_failure_is_tagged(): + """When the orchestration loop exceeds max_uses (the executor keeps calling + the advisor), the AdvisorMaxIterationsError must be tagged so the healthy + executor deployment is not cooled down.""" + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import ( + AdvisorMaxIterationsError, + AdvisorOrchestrationHandler, + ) + from litellm.router_utils.cooldown_handlers import is_advisor_orchestration_failure + + advisor_tool_with_max = {**ADVISOR_TOOL, "max_uses": 1} + + async def mock_call(model, messages, tools, stream, max_tokens, **kwargs): + # Executor always asks for the advisor; advisor always succeeds, so the + # loop is driven purely by max_uses rather than any deployment failure. + if tools is None: + return _make_text_response("Here is my advice.") + return _make_advisor_tool_use_response() + + with patch( + "litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler", + side_effect=mock_call, + ): + h = AdvisorOrchestrationHandler() + with pytest.raises(AdvisorMaxIterationsError) as exc_info: + await h.handle( + model="openai/gpt-4o-mini", + messages=MESSAGES, + tools=[advisor_tool_with_max], + stream=False, + max_tokens=512, + custom_llm_provider="openai", + ) + + assert is_advisor_orchestration_failure(exc_info.value) is True + + +@pytest.mark.asyncio +async def test_executor_failure_is_not_tagged(): + """A failure of the executor call (not advisor orchestration) must NOT be + tagged — the selected deployment genuinely failed and should cool down.""" + import litellm + from litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor import ( + AdvisorOrchestrationHandler, + ) + from litellm.router_utils.cooldown_handlers import is_advisor_orchestration_failure + + async def mock_call(model, messages, tools, stream, max_tokens, **kwargs): + raise litellm.AuthenticationError( # executor (first call) fails + message="invalid deployment credentials", + llm_provider="openai", + model=model, + ) + + with patch( + "litellm.llms.anthropic.experimental_pass_through.messages.interceptors.advisor._call_messages_handler", + side_effect=mock_call, + ): + h = AdvisorOrchestrationHandler() + with pytest.raises(litellm.AuthenticationError) as exc_info: + await h.handle( + model="openai/gpt-4o-mini", + messages=MESSAGES, + tools=[ADVISOR_TOOL], + stream=False, + max_tokens=512, + custom_llm_provider="openai", + ) + + assert is_advisor_orchestration_failure(exc_info.value) is False diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index cffc3dd0aba..a9e5b3316e0 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -5730,6 +5730,76 @@ class TestRouterRequestTimeoutPropagation: ) +class TestAdvisorSubCallCooldown: + """Regression for LIT-4565: an advisor orchestration failure must not cool + down the selected (healthy) deployment, which would reject unrelated + callers to the same model group.""" + + def _router(self): + return litellm.Router( + model_list=[ + { + "model_name": "claude-sonnet-5", + "litellm_params": {"model": "bedrock/us.anthropic.claude-opus-4-8"}, + "model_info": {"id": "dep-1"}, + } + ], + ) + + def _kwargs(self, exception): + return { + "exception": exception, + "litellm_params": {"model_info": {"id": "dep-1"}, "metadata": {}}, + } + + def _auth_error(self): + return litellm.AuthenticationError( + message="x-api-key header is required", + llm_provider="anthropic", + model="claude-opus-4-8", + ) + + def _cooled_down_ids(self, router): + active = router.cooldown_cache.get_active_cooldowns( + model_ids=["dep-1"], parent_otel_span=None + ) + return [entry[0] for entry in active] + + @pytest.mark.asyncio + async def test_untagged_auth_error_cools_down_deployment(self): + from datetime import datetime + + router = self._router() + now = datetime.now() + assert ( + router.deployment_callback_on_failure( + self._kwargs(self._auth_error()), None, now, now + ) + is True + ) + assert "dep-1" in self._cooled_down_ids(router) + + def test_advisor_orchestration_failure_does_not_cool_down_deployment(self): + from datetime import datetime + + from litellm.router_utils.cooldown_handlers import ( + mark_advisor_orchestration_failure, + ) + + router = self._router() + exception = self._auth_error() + mark_advisor_orchestration_failure(exception) + + now = datetime.now() + assert ( + router.deployment_callback_on_failure( + self._kwargs(exception), None, now, now + ) + is False + ) + assert "dep-1" not in self._cooled_down_ids(router) + + def test_get_configured_token_limits_reads_deployment_model_info(): router = litellm.Router( model_list=[