diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 490912d42eb..750ac0aeac8 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -22,7 +22,6 @@ from litellm.constants import ( DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET, RESPONSE_FORMAT_TOOL_NAME, ) -from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.litellm_core_utils.prompt_templates.common_utils import ( sanitize_input_schema_for_anthropic, ) @@ -67,7 +66,6 @@ from litellm.types.llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionToolCallFunctionChunk, ChatCompletionToolParam, - OpenAIChatCompletionFinishReason, OpenAIMcpServerTool, OpenAIWebSearchOptions, ) @@ -81,6 +79,7 @@ from litellm.types.utils import ( CompletionTokensDetailsWrapper, PromptTokensDetailsWrapper, ServerToolUse, + map_finish_reason_and_stash_native, ) from litellm.types.utils import Message as LitellmMessage from litellm.utils import ( @@ -2667,10 +2666,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): model_response.choices[0].message = _message model_response._hidden_params["original_response"] = completion_response["content"] - model_response.choices[0].finish_reason = cast( - OpenAIChatCompletionFinishReason, - map_finish_reason(completion_response["stop_reason"]), + _choice = model_response.choices[0] + _mapped_reason, _provider_specific_fields = map_finish_reason_and_stash_native( + completion_response["stop_reason"], getattr(_choice, "provider_specific_fields", None) ) + _choice.finish_reason = _mapped_reason + if _provider_specific_fields is not None: + setattr(_choice, "provider_specific_fields", _provider_specific_fields) usage: Final = self.calculate_usage( usage_object=completion_response["usage"], diff --git a/litellm/router.py b/litellm/router.py index 24554e61516..7465853f2e2 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -760,6 +760,24 @@ def as_output_cap(value: object) -> int | None: return cap if cap >= 0 else None +## Exception classes a treat_finish_reason_as_failure value may name: resolved from litellm at +## use time, validated at Router construction. +_FINISH_REASON_FAILURE_EXCEPTION_NAMES: Final = frozenset( + { + "RateLimitError", + "APIError", + "BadRequestError", + "Timeout", + "ServiceUnavailableError", + "InternalServerError", + } +) + +## Healthy terminal reasons in the mapped OpenAI set: keys of treat_finish_reason_as_failure that +## name one of these would fail successful responses, so construction warns about them. +_HEALTHY_TERMINAL_FINISH_REASONS: Final = frozenset(("stop", "length", "tool_calls", "function_call")) + + class Router: model_names: set = set() cache_responses: bool | None = False @@ -804,6 +822,7 @@ class Router: fallbacks: list = [], context_window_fallbacks: list = [], content_policy_fallbacks: list = [], + treat_finish_reason_as_failure: Mapping[str, str] | None = None, model_group_alias: dict[str, str | RouterModelGroupAliasItem] | None = {}, enable_pre_call_checks: bool = False, enable_tag_filtering: bool = False, @@ -1118,6 +1137,28 @@ class Router: _content_policy_fallbacks: Final = content_policy_fallbacks or litellm.content_policy_fallbacks self.validate_fallbacks(fallback_param=_content_policy_fallbacks) self.content_policy_fallbacks = _content_policy_fallbacks + + ## treat_finish_reason_as_failure: map a terminal finish/stop reason on a 200 response to a + ## router-understood exception class, so the mapped reason engages allowed_fails/cooldowns/ + ## fallbacks like any failure. Reason strings are matched exactly. + if treat_finish_reason_as_failure is not None: + for exception_name in treat_finish_reason_as_failure.values(): + if exception_name not in _FINISH_REASON_FAILURE_EXCEPTION_NAMES: + raise ValueError( + f"treat_finish_reason_as_failure values must be one of {sorted(_FINISH_REASON_FAILURE_EXCEPTION_NAMES)}, got {exception_name}" + ) + self.treat_finish_reason_as_failure = treat_finish_reason_as_failure + if treat_finish_reason_as_failure: + verbose_router_logger.warning( + "treat_finish_reason_as_failure applies to non-streaming responses only; a streamed 200 with the mapped stop reason is delivered unchanged." + ) + healthy_terminal_keys: Final = treat_finish_reason_as_failure.keys() & _HEALTHY_TERMINAL_FINISH_REASONS + if healthy_terminal_keys: + verbose_router_logger.warning( + "treat_finish_reason_as_failure keys %s are healthy terminal reasons in the mapped OpenAI set; mapping them fails successful responses. Keys are matched against provider-native stop reasons.", + sorted(healthy_terminal_keys), + ) + self.total_calls: defaultdict = defaultdict(int) # dict to store total calls made to each model self.fail_calls: defaultdict = defaultdict(int) # dict to store fail_calls made to each model self.success_calls: defaultdict = defaultdict(int) # dict to store success_calls made to each model @@ -2618,6 +2659,14 @@ class Router: llm_provider="", ) + ## CHECK MAPPED FINISH REASON ERROR ## + if isinstance(response, ModelResponse): + _mapped_reason: Final = self._get_mapped_finish_reason(response) + if _mapped_reason is not None: + self._handle_mapped_finish_reason_failure( + model=model, deployment=deployment, reason=_mapped_reason, kwargs=kwargs + ) + if ( isinstance(response, CustomStreamWrapper) and response.completion_stream is None @@ -3777,6 +3826,14 @@ class Router: llm_provider="", ) + ## CHECK MAPPED FINISH REASON ERROR ## + if isinstance(response, ModelResponse): + _mapped_reason: Final = self._get_mapped_finish_reason(response) + if _mapped_reason is not None: + self._handle_mapped_finish_reason_failure( + model=model, deployment=deployment, reason=_mapped_reason, kwargs=kwargs + ) + if ( isinstance(response, CustomStreamWrapper) and response.completion_stream is None @@ -5389,6 +5446,17 @@ class Router: refusal_details: Final = cast(dict, response["stop_details"]) # cast-ok: gate verified the shape raise safeguard_refusal_error(model=model, stop_details=refusal_details) + if ( + self.treat_finish_reason_as_failure + and getattr(original_generic_function, "__name__", "") == "anthropic_messages" + and isinstance(response, dict) + ): + stop_reason: Final = response.get("stop_reason") + if stop_reason in self.treat_finish_reason_as_failure: + self._handle_mapped_finish_reason_failure( + model=model, deployment=deployment, reason=stop_reason, kwargs=kwargs + ) + self.success_calls[model_name] += 1 verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -8466,6 +8534,37 @@ class Router: content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks) if content_policy_fallbacks is not None: return self._has_content_policy_fallback(model_group, kwargs) + return self._generic_fallback_available(model_group, kwargs) + + def _get_mapped_finish_reason(self, response: ModelResponse) -> str | None: + """ + The finish reason configured in treat_finish_reason_as_failure that this response carries, + or None. Checks both the mapped finish_reason and the pre-mapping value stashed in + provider_specific_fields["native_finish_reason"]. Streaming detection is a follow-up + modeled on _aanthropic_messages_streaming_iterator. + """ + if not self.treat_finish_reason_as_failure: + return None + if not (response.choices and len(response.choices) > 0): + return None + choice: Final = response.choices[0] + if choice.finish_reason in self.treat_finish_reason_as_failure: + return choice.finish_reason + _provider_specific_fields: Final = getattr(choice, "provider_specific_fields", None) + native_reason: Final = ( + _provider_specific_fields.get("native_finish_reason") if _provider_specific_fields else None + ) + if native_reason in self.treat_finish_reason_as_failure: + return native_reason + return None + + def _generic_fallback_available(self, model_group: str, kwargs: Mapping[str, Any]) -> bool: + """ + Whether a generic fallback can serve a retry: default fallbacks set, or a generic chain + resolving for this request. Shared tail of the fallback-availability gates. + """ + if fallbacks_disabled_for_request(kwargs): + return False if self._has_default_fallbacks(): return True fallbacks: Final = kwargs.get("fallbacks", self.fallbacks) @@ -8527,6 +8626,73 @@ class Router: ) return has_unattempted_fallback_target(resolved, kwargs) + def _handle_mapped_finish_reason_failure( + self, model: str, deployment: Mapping[str, Any], reason: str, kwargs: Mapping[str, Any] + ) -> None: + """ + Account for a mapped finish-reason failure, then raise the configured exception into the + fallback chain when a generic fallback can serve. Accounting happens before the gate: + the raise lands after the 200 came back, so litellm's failure callbacks never fire for + it, and this is the only path that parks the deployment. A deployment with no model_info + id cannot be accounted or parked, but the raise still applies to it. + """ + exception: Final = self._account_mapped_finish_reason_failure( + model=model, deployment=deployment, reason=reason, kwargs=kwargs + ) or self._finish_reason_failure_error(model=model, reason=reason) + if self._generic_fallback_available(model, kwargs): + raise exception + + def _finish_reason_failure_error(self, model: str, reason: str) -> Exception: + """Build the exception instance configured for a mapped finish reason.""" + message: Final = f"Response finished with reason '{reason}' (treat_finish_reason_as_failure)." + finish_reason_map: Final = self.treat_finish_reason_as_failure + if finish_reason_map is None: + return litellm.APIError(status_code=500, message=message, llm_provider="", model=model) + exception_name: Final = finish_reason_map[reason] + exception_cls: Final = getattr(litellm, exception_name) + if exception_name == "APIError": + return exception_cls(status_code=500, message=message, llm_provider="", model=model) + return exception_cls(message=message, llm_provider="", model=model) + + def _account_mapped_finish_reason_failure( + self, model: str, deployment: Mapping[str, Any], reason: str, kwargs: Mapping[str, Any] + ) -> Exception | None: + """ + Count and park a mapped finish-reason failure: increment the per-minute failure counter + and set the cooldown, honoring a deployment-level cooldown_time like + deployment_callback_on_failure does (the retry-after-header tier has no counterpart + here: the exception is synthesized, it carries no response headers). Returns the built + exception so the caller can raise the same instance it accounted for, or None when the + deployment has no id to account against. + """ + raw_model_info: Final = deployment.get("model_info") + model_info: Final = raw_model_info if isinstance(raw_model_info, dict) else None + deployment_id: Final = model_info.get("id") if model_info is not None else None + if deployment_id is None: + return None + raw_litellm_params: Final = deployment.get("litellm_params") + litellm_params: Final = raw_litellm_params if isinstance(raw_litellm_params, dict) else None + deployment_cooldown: Final = _first_present(model_info, litellm_params, key="cooldown_time") + time_to_cooldown: Final = ( + deployment_cooldown if deployment_cooldown is not None and deployment_cooldown >= 0 else self.cooldown_time + ) + exception: Final = self._finish_reason_failure_error(model=model, reason=reason) + litellm_metadata: Final = get_litellm_metadata_from_kwargs(kwargs) + requested_model_group: Final = litellm_metadata.get("model_group") if litellm_metadata else None + increment_deployment_failures_for_current_minute( + litellm_router_instance=self, + deployment_id=deployment_id, + ) + _set_cooldown_deployments( + litellm_router_instance=self, + exception_status=exception.status_code, + original_exception=exception, + deployment=deployment_id, + time_to_cooldown=time_to_cooldown, + requested_model_group=requested_model_group, + ) + return exception + def _should_raise_content_policy_error(self, model: str, response: ModelResponse, kwargs: dict) -> bool: """ Determines if a content policy error should be raised. diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6919fd6fd27..65faf5642fd 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -1630,6 +1630,20 @@ class Delta(SafeAttributeModel, OpenAIObject): setattr(self, key, value) +def map_finish_reason_and_stash_native( + finish_reason: str, provider_specific_fields: Mapping[str, Any] | None +) -> tuple[OpenAIChatCompletionFinishReason, dict[str, Any] | None]: # mutable-ok: callers extend the returned stash + """Map a provider-native finish reason to the OpenAI set; when the native value differs + from the mapped one, preserve it under provider_specific_fields["native_finish_reason"] + so downstream consumers can still see what the provider actually sent.""" + mapped: Final = map_finish_reason(finish_reason) + if finish_reason == mapped: + return mapped, provider_specific_fields + stash: Final = dict(provider_specific_fields or ()) # mutable-ok: the stash must stay a plain extensible dict + stash["native_finish_reason"] = finish_reason + return mapped, stash + + class Choices(SafeAttributeModel, OpenAIObject): finish_reason: OpenAIChatCompletionFinishReason index: int @@ -1649,11 +1663,10 @@ class Choices(SafeAttributeModel, OpenAIObject): **params, ) -> None: if finish_reason is not None: - mapped: Final = map_finish_reason(finish_reason) + mapped, provider_specific_fields = map_finish_reason_and_stash_native( + finish_reason, provider_specific_fields + ) params["finish_reason"] = mapped - if finish_reason != mapped: - provider_specific_fields = dict(provider_specific_fields) if provider_specific_fields else {} - provider_specific_fields["native_finish_reason"] = finish_reason else: params["finish_reason"] = "stop" if index is not None: diff --git a/tests/router_unit_tests/test_router_finish_reason_failure.py b/tests/router_unit_tests/test_router_finish_reason_failure.py new file mode 100644 index 00000000000..17e728e992e --- /dev/null +++ b/tests/router_unit_tests/test_router_finish_reason_failure.py @@ -0,0 +1,220 @@ +""" +Unit tests for the treat_finish_reason_as_failure router knob. + +A provider can report a terminal condition (context window exceeded, and +similar) as a stop reason on an HTTP 200. The knob maps such reasons to a +router-understood exception class, so the mapped reason engages allowed_fails, +cooldowns, and fallbacks like any failure. When the mapped reason is present +but no generic fallback can serve the retry, the response reaches the client +unchanged while the deployment still counts the failure. +""" + +import json +from typing import Any + +import httpx +import pytest +from pytest import MonkeyPatch + +import litellm +from litellm import Router +from litellm.router_utils.cooldown_handlers import _get_cooldown_deployments + +CONTEXT_WINDOW_RESPONSE: dict[str, Any] = { + "id": "msg_context", + "type": "message", + "role": "assistant", + "model": "claude-fable-5", + "content": [], + "stop_reason": "model_context_window_exceeded", + "stop_sequence": None, + "usage": {"input_tokens": 25, "output_tokens": 1}, +} + +OK_RESPONSE: dict[str, Any] = { + "id": "msg_ok", + "type": "message", + "role": "assistant", + "model": "claude-opus-5", + "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 25, "output_tokens": 2}, +} + + +class FakeAnthropicUpstream: + """Intercepts the third-party transport (httpx.AsyncClient.send): reports the + context-window stop reason on fable models, answers on others. The router + deliberately does not forward caller-injected clients, so the transport is the + seam that exercises the real litellm pipeline end to end.""" + + def __init__(self) -> None: + self.calls: list[str] = [] + + async def send(self, request: httpx.Request, **kwargs: Any) -> httpx.Response: + body = json.loads(request.content or b"{}") + model = body.get("model", "") + self.calls.append(model) + overrun = "fable" in model + return httpx.Response( + 200, + json=CONTEXT_WINDOW_RESPONSE if overrun else OK_RESPONSE, + request=request, + ) + + def install(self, monkeypatch: MonkeyPatch) -> None: + async def _send(_client: httpx.AsyncClient, request: httpx.Request, **kwargs: Any) -> httpx.Response: + return await self.send(request, **kwargs) + + monkeypatch.setattr(httpx.AsyncClient, "send", _send) + + +FABLE_TIER = { + "model_name": "fable-tier", + "litellm_params": {"model": "anthropic/claude-fable-5", "api_key": "sk-test"}, +} +OPUS_TARGET = { + "model_name": "opus-target", + "litellm_params": {"model": "anthropic/claude-opus-5", "api_key": "sk-test"}, +} + + +def _knob() -> dict[str, str]: + return {"model_context_window_exceeded": "RateLimitError"} + + +def _deployment_id(router: Router, index: int = 0) -> str: + return router.model_list[index]["model_info"]["id"] + + +@pytest.mark.asyncio +async def test_chat_completion_mapped_reason_falls_back_and_cools_down(monkeypatch: MonkeyPatch): + fake = FakeAnthropicUpstream() + router = Router( + model_list=[FABLE_TIER, OPUS_TARGET], + treat_finish_reason_as_failure=_knob(), + default_fallbacks=["opus-target"], + num_retries=0, + allowed_fails=0, + cooldown_time=10, + ) + fake.install(monkeypatch) + + response = await router.acompletion(model="fable-tier", max_tokens=16, messages=[{"role": "user", "content": "hi"}]) + + assert response.model == "claude-opus-5" + assert len(fake.calls) == 2 + assert "claude-fable-5" in fake.calls[0] + assert "claude-opus-5" in fake.calls[1] + assert router.fail_calls["anthropic/claude-fable-5"] == 1 + fable_id = _deployment_id(router, 0) + assert fable_id in _get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) + + +@pytest.mark.asyncio +async def test_anthropic_messages_mapped_reason_falls_back(monkeypatch: MonkeyPatch): + fake = FakeAnthropicUpstream() + router = Router( + model_list=[FABLE_TIER, OPUS_TARGET], + treat_finish_reason_as_failure=_knob(), + default_fallbacks=["opus-target"], + num_retries=0, + allowed_fails=0, + cooldown_time=10, + ) + fake.install(monkeypatch) + + response = await router.aanthropic_messages( + model="fable-tier", max_tokens=16, messages=[{"role": "user", "content": "hi"}] + ) + + assert response["id"] == "msg_ok" + assert response["stop_reason"] == "end_turn" + assert len(fake.calls) == 2 + assert "claude-opus-5" in fake.calls[1] + + +@pytest.mark.asyncio +async def test_mapped_reason_without_fallback_returns_response_and_counts_failure(monkeypatch: MonkeyPatch): + fake = FakeAnthropicUpstream() + router = Router( + model_list=[FABLE_TIER], + treat_finish_reason_as_failure=_knob(), + num_retries=0, + allowed_fails=0, + cooldown_time=10, + ) + fake.install(monkeypatch) + + response = await router.acompletion(model="fable-tier", max_tokens=16, messages=[{"role": "user", "content": "hi"}]) + + assert response.model == "claude-fable-5" + assert len(fake.calls) == 1 + fable_id = _deployment_id(router, 0) + failures = router.cache.get_cache(local_only=True, key=f"{fable_id}:fails") + assert failures == 1 + assert fable_id in _get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None) + + +@pytest.mark.asyncio +async def test_knob_unset_ignores_terminal_stop_reason(monkeypatch: MonkeyPatch): + fake = FakeAnthropicUpstream() + router = Router(model_list=[FABLE_TIER], num_retries=0) + fake.install(monkeypatch) + + response = await router.acompletion(model="fable-tier", max_tokens=16, messages=[{"role": "user", "content": "hi"}]) + + assert response.model == "claude-fable-5" + assert len(fake.calls) == 1 + assert router.fail_calls["fable-tier"] == 0 + + +def test_unknown_exception_name_raises_at_construction(): + with pytest.raises(ValueError, match="NotAnException"): + Router(model_list=[], treat_finish_reason_as_failure={"x": "NotAnException"}) + + +def test_healthy_terminal_key_warns_at_construction(capsys: pytest.CaptureFixture): + Router(model_list=[], treat_finish_reason_as_failure={"stop": "RateLimitError"}) + logged = capsys.readouterr().err + capsys.readouterr().out + assert "healthy terminal reasons" in logged + + +@pytest.mark.asyncio +async def test_mapped_finish_reason_helpers_direct(monkeypatch: MonkeyPatch): + """Direct coverage of the knob helpers (the router code-coverage check matches by name).""" + fake = FakeAnthropicUpstream() + router = Router( + model_list=[FABLE_TIER, OPUS_TARGET], + treat_finish_reason_as_failure=_knob(), + default_fallbacks=["opus-target"], + num_retries=0, + allowed_fails=0, + cooldown_time=10, + ) + fake.install(monkeypatch) + + ok = await router.acompletion(model="opus-target", max_tokens=16, messages=[{"role": "user", "content": "hi"}]) + assert router._get_mapped_finish_reason(ok) is None + assert router._generic_fallback_available("fable-tier", {}) is True + + error = router._finish_reason_failure_error(model="fable-tier", reason="model_context_window_exceeded") + assert error.status_code == 429 + + deployment = router.model_list[0] + accounted = router._account_mapped_finish_reason_failure( + model="fable-tier", + deployment=deployment, + reason="model_context_window_exceeded", + kwargs={}, + ) + assert accounted is not None + + with pytest.raises(litellm.RateLimitError): + router._handle_mapped_finish_reason_failure( + model="fable-tier", + deployment=deployment, + reason="model_context_window_exceeded", + kwargs={}, + )