scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 109

Merge 8c59d2550b into e768ad55ce

Browse source
This commit is contained in:
Paolo Antinori 2026-10-03 21:12:11 +09:00 • committed by GitHub
commit 606f8c8ef9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 410 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
litellm/llms/anthropic/chat/transformation.py
View file

@ -22,7 +22,6 @@ from litellm.constants import (
DEFAULT_REASONING_EFFORT_XHIGH_THINKING_BUDGET,
RESPONSE_FORMAT_TOOL_NAME,
)
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.prompt_templates.common_utils import (
sanitize_input_schema_for_anthropic,
)
@ -67,7 +66,6 @@ from litellm.types.llms.openai import (
ChatCompletionToolCallChunk,
ChatCompletionToolCallFunctionChunk,
ChatCompletionToolParam,
OpenAIChatCompletionFinishReason,
OpenAIMcpServerTool,
OpenAIWebSearchOptions,
)
@ -81,6 +79,7 @@ from litellm.types.utils import (
CompletionTokensDetailsWrapper,
PromptTokensDetailsWrapper,
ServerToolUse,
map_finish_reason_and_stash_native,
)
from litellm.types.utils import Message as LitellmMessage
from litellm.utils import (
@ -2667,10 +2666,13 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
model_response.choices[0].message = _message
model_response._hidden_params["original_response"] = completion_response["content"]
model_response.choices[0].finish_reason = cast(
OpenAIChatCompletionFinishReason,
map_finish_reason(completion_response["stop_reason"]),
_choice = model_response.choices[0]
_mapped_reason, _provider_specific_fields = map_finish_reason_and_stash_native(
completion_response["stop_reason"], getattr(_choice, "provider_specific_fields", None)
)
_choice.finish_reason = _mapped_reason
if _provider_specific_fields is not None:
setattr(_choice, "provider_specific_fields", _provider_specific_fields)
usage: Final = self.calculate_usage(
usage_object=completion_response["usage"],

166
litellm/router.py
View file

@ -760,6 +760,24 @@ def as_output_cap(value: object) -> int | None:
return cap if cap >= 0 else None
## Exception classes a treat_finish_reason_as_failure value may name: resolved from litellm at
## use time, validated at Router construction.
_FINISH_REASON_FAILURE_EXCEPTION_NAMES: Final = frozenset(
{
"RateLimitError",
"APIError",
"BadRequestError",
"Timeout",
"ServiceUnavailableError",
"InternalServerError",
}
)
## Healthy terminal reasons in the mapped OpenAI set: keys of treat_finish_reason_as_failure that
## name one of these would fail successful responses, so construction warns about them.
_HEALTHY_TERMINAL_FINISH_REASONS: Final = frozenset(("stop", "length", "tool_calls", "function_call"))
class Router:
model_names: set = set()
cache_responses: bool | None = False
@ -804,6 +822,7 @@ class Router:
fallbacks: list = [],
context_window_fallbacks: list = [],
content_policy_fallbacks: list = [],
treat_finish_reason_as_failure: Mapping[str, str] | None = None,
model_group_alias: dict[str, str | RouterModelGroupAliasItem] | None = {},
enable_pre_call_checks: bool = False,
enable_tag_filtering: bool = False,
@ -1118,6 +1137,28 @@ class Router:
_content_policy_fallbacks: Final = content_policy_fallbacks or litellm.content_policy_fallbacks
self.validate_fallbacks(fallback_param=_content_policy_fallbacks)
self.content_policy_fallbacks = _content_policy_fallbacks
## treat_finish_reason_as_failure: map a terminal finish/stop reason on a 200 response to a
## router-understood exception class, so the mapped reason engages allowed_fails/cooldowns/
## fallbacks like any failure. Reason strings are matched exactly.
if treat_finish_reason_as_failure is not None:
for exception_name in treat_finish_reason_as_failure.values():
if exception_name not in _FINISH_REASON_FAILURE_EXCEPTION_NAMES:
raise ValueError(
f"treat_finish_reason_as_failure values must be one of {sorted(_FINISH_REASON_FAILURE_EXCEPTION_NAMES)}, got {exception_name}"
)
self.treat_finish_reason_as_failure = treat_finish_reason_as_failure
if treat_finish_reason_as_failure:
verbose_router_logger.warning(
"treat_finish_reason_as_failure applies to non-streaming responses only; a streamed 200 with the mapped stop reason is delivered unchanged."
)
healthy_terminal_keys: Final = treat_finish_reason_as_failure.keys() & _HEALTHY_TERMINAL_FINISH_REASONS
if healthy_terminal_keys:
verbose_router_logger.warning(
"treat_finish_reason_as_failure keys %s are healthy terminal reasons in the mapped OpenAI set; mapping them fails successful responses. Keys are matched against provider-native stop reasons.",
sorted(healthy_terminal_keys),
)
self.total_calls: defaultdict = defaultdict(int) # dict to store total calls made to each model
self.fail_calls: defaultdict = defaultdict(int) # dict to store fail_calls made to each model
self.success_calls: defaultdict = defaultdict(int) # dict to store success_calls made to each model
@ -2618,6 +2659,14 @@ class Router:
llm_provider="",
)
## CHECK MAPPED FINISH REASON ERROR ##
if isinstance(response, ModelResponse):
_mapped_reason: Final = self._get_mapped_finish_reason(response)
if _mapped_reason is not None:
self._handle_mapped_finish_reason_failure(
model=model, deployment=deployment, reason=_mapped_reason, kwargs=kwargs
)
if (
isinstance(response, CustomStreamWrapper)
and response.completion_stream is None
@ -3777,6 +3826,14 @@ class Router:
llm_provider="",
)
## CHECK MAPPED FINISH REASON ERROR ##
if isinstance(response, ModelResponse):
_mapped_reason: Final = self._get_mapped_finish_reason(response)
if _mapped_reason is not None:
self._handle_mapped_finish_reason_failure(
model=model, deployment=deployment, reason=_mapped_reason, kwargs=kwargs
)
if (
isinstance(response, CustomStreamWrapper)
and response.completion_stream is None
@ -5389,6 +5446,17 @@ class Router:
refusal_details: Final = cast(dict, response["stop_details"]) # cast-ok: gate verified the shape
raise safeguard_refusal_error(model=model, stop_details=refusal_details)
if (
self.treat_finish_reason_as_failure
and getattr(original_generic_function, "__name__", "") == "anthropic_messages"
and isinstance(response, dict)
):
stop_reason: Final = response.get("stop_reason")
if stop_reason in self.treat_finish_reason_as_failure:
self._handle_mapped_finish_reason_failure(
model=model, deployment=deployment, reason=stop_reason, kwargs=kwargs
)
self.success_calls[model_name] += 1
verbose_router_logger.info("ageneric_api_call_with_fallbacks(model=%s)\x1b[32m 200 OK\x1b[0m", model_name)
@ -8466,6 +8534,37 @@ class Router:
content_policy_fallbacks: Final = kwargs.get("content_policy_fallbacks", self.content_policy_fallbacks)
if content_policy_fallbacks is not None:
return self._has_content_policy_fallback(model_group, kwargs)
return self._generic_fallback_available(model_group, kwargs)
def _get_mapped_finish_reason(self, response: ModelResponse) -> str | None:
"""
The finish reason configured in treat_finish_reason_as_failure that this response carries,
or None. Checks both the mapped finish_reason and the pre-mapping value stashed in
provider_specific_fields["native_finish_reason"]. Streaming detection is a follow-up
modeled on _aanthropic_messages_streaming_iterator.
"""
if not self.treat_finish_reason_as_failure:
return None
if not (response.choices and len(response.choices) > 0):
return None
choice: Final = response.choices[0]
if choice.finish_reason in self.treat_finish_reason_as_failure:
return choice.finish_reason
_provider_specific_fields: Final = getattr(choice, "provider_specific_fields", None)
native_reason: Final = (
_provider_specific_fields.get("native_finish_reason") if _provider_specific_fields else None
)
if native_reason in self.treat_finish_reason_as_failure:
return native_reason
return None
def _generic_fallback_available(self, model_group: str, kwargs: Mapping[str, Any]) -> bool:
"""
Whether a generic fallback can serve a retry: default fallbacks set, or a generic chain
resolving for this request. Shared tail of the fallback-availability gates.
"""
if fallbacks_disabled_for_request(kwargs):
return False
if self._has_default_fallbacks():
return True
fallbacks: Final = kwargs.get("fallbacks", self.fallbacks)
@ -8527,6 +8626,73 @@ class Router:
)
return has_unattempted_fallback_target(resolved, kwargs)
def _handle_mapped_finish_reason_failure(
self, model: str, deployment: Mapping[str, Any], reason: str, kwargs: Mapping[str, Any]
) -> None:
"""
Account for a mapped finish-reason failure, then raise the configured exception into the
fallback chain when a generic fallback can serve. Accounting happens before the gate:
the raise lands after the 200 came back, so litellm's failure callbacks never fire for
it, and this is the only path that parks the deployment. A deployment with no model_info
id cannot be accounted or parked, but the raise still applies to it.
"""
exception: Final = self._account_mapped_finish_reason_failure(
model=model, deployment=deployment, reason=reason, kwargs=kwargs
) or self._finish_reason_failure_error(model=model, reason=reason)
if self._generic_fallback_available(model, kwargs):
raise exception
def _finish_reason_failure_error(self, model: str, reason: str) -> Exception:
"""Build the exception instance configured for a mapped finish reason."""
message: Final = f"Response finished with reason '{reason}' (treat_finish_reason_as_failure)."
finish_reason_map: Final = self.treat_finish_reason_as_failure
if finish_reason_map is None:
return litellm.APIError(status_code=500, message=message, llm_provider="", model=model)
exception_name: Final = finish_reason_map[reason]
exception_cls: Final = getattr(litellm, exception_name)
if exception_name == "APIError":
return exception_cls(status_code=500, message=message, llm_provider="", model=model)
return exception_cls(message=message, llm_provider="", model=model)
def _account_mapped_finish_reason_failure(
self, model: str, deployment: Mapping[str, Any], reason: str, kwargs: Mapping[str, Any]
) -> Exception | None:
"""
Count and park a mapped finish-reason failure: increment the per-minute failure counter
and set the cooldown, honoring a deployment-level cooldown_time like
deployment_callback_on_failure does (the retry-after-header tier has no counterpart
here: the exception is synthesized, it carries no response headers). Returns the built
exception so the caller can raise the same instance it accounted for, or None when the
deployment has no id to account against.
"""
raw_model_info: Final = deployment.get("model_info")
model_info: Final = raw_model_info if isinstance(raw_model_info, dict) else None
deployment_id: Final = model_info.get("id") if model_info is not None else None
if deployment_id is None:
return None
raw_litellm_params: Final = deployment.get("litellm_params")
litellm_params: Final = raw_litellm_params if isinstance(raw_litellm_params, dict) else None
deployment_cooldown: Final = _first_present(model_info, litellm_params, key="cooldown_time")
time_to_cooldown: Final = (
deployment_cooldown if deployment_cooldown is not None and deployment_cooldown >= 0 else self.cooldown_time
)
exception: Final = self._finish_reason_failure_error(model=model, reason=reason)
litellm_metadata: Final = get_litellm_metadata_from_kwargs(kwargs)
requested_model_group: Final = litellm_metadata.get("model_group") if litellm_metadata else None
increment_deployment_failures_for_current_minute(
litellm_router_instance=self,
deployment_id=deployment_id,
)
_set_cooldown_deployments(
litellm_router_instance=self,
exception_status=exception.status_code,
original_exception=exception,
deployment=deployment_id,
time_to_cooldown=time_to_cooldown,
requested_model_group=requested_model_group,
)
return exception
def _should_raise_content_policy_error(self, model: str, response: ModelResponse, kwargs: dict) -> bool:
"""
Determines if a content policy error should be raised.

21
litellm/types/utils.py
View file

@ -1630,6 +1630,20 @@ class Delta(SafeAttributeModel, OpenAIObject):
setattr(self, key, value)
def map_finish_reason_and_stash_native(
finish_reason: str, provider_specific_fields: Mapping[str, Any] | None
) -> tuple[OpenAIChatCompletionFinishReason, dict[str, Any] | None]: # mutable-ok: callers extend the returned stash
"""Map a provider-native finish reason to the OpenAI set; when the native value differs
from the mapped one, preserve it under provider_specific_fields["native_finish_reason"]
so downstream consumers can still see what the provider actually sent."""
mapped: Final = map_finish_reason(finish_reason)
if finish_reason == mapped:
return mapped, provider_specific_fields
stash: Final = dict(provider_specific_fields or ()) # mutable-ok: the stash must stay a plain extensible dict
stash["native_finish_reason"] = finish_reason
return mapped, stash
class Choices(SafeAttributeModel, OpenAIObject):
finish_reason: OpenAIChatCompletionFinishReason
index: int
@ -1649,11 +1663,10 @@ class Choices(SafeAttributeModel, OpenAIObject):
**params,
) -> None:
if finish_reason is not None:
mapped: Final = map_finish_reason(finish_reason)
mapped, provider_specific_fields = map_finish_reason_and_stash_native(
finish_reason, provider_specific_fields
)
params["finish_reason"] = mapped
if finish_reason != mapped:
provider_specific_fields = dict(provider_specific_fields) if provider_specific_fields else {}
provider_specific_fields["native_finish_reason"] = finish_reason
else:
params["finish_reason"] = "stop"
if index is not None:

220
tests/router_unit_tests/test_router_finish_reason_failure.py Normal file
View file

@ -0,0 +1,220 @@
"""
Unit tests for the treat_finish_reason_as_failure router knob.
A provider can report a terminal condition (context window exceeded, and
similar) as a stop reason on an HTTP 200. The knob maps such reasons to a
router-understood exception class, so the mapped reason engages allowed_fails,
cooldowns, and fallbacks like any failure. When the mapped reason is present
but no generic fallback can serve the retry, the response reaches the client
unchanged while the deployment still counts the failure.
"""
import json
from typing import Any
import httpx
import pytest
from pytest import MonkeyPatch
import litellm
from litellm import Router
from litellm.router_utils.cooldown_handlers import _get_cooldown_deployments
CONTEXT_WINDOW_RESPONSE: dict[str, Any] = {
"id": "msg_context",
"type": "message",
"role": "assistant",
"model": "claude-fable-5",
"content": [],
"stop_reason": "model_context_window_exceeded",
"stop_sequence": None,
"usage": {"input_tokens": 25, "output_tokens": 1},
}
OK_RESPONSE: dict[str, Any] = {
"id": "msg_ok",
"type": "message",
"role": "assistant",
"model": "claude-opus-5",
"content": [{"type": "text", "text": "hello"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 25, "output_tokens": 2},
}
class FakeAnthropicUpstream:
"""Intercepts the third-party transport (httpx.AsyncClient.send): reports the
context-window stop reason on fable models, answers on others. The router
deliberately does not forward caller-injected clients, so the transport is the
seam that exercises the real litellm pipeline end to end."""
def __init__(self) -> None:
self.calls: list[str] = []
async def send(self, request: httpx.Request, **kwargs: Any) -> httpx.Response:
body = json.loads(request.content or b"{}")
model = body.get("model", "")
self.calls.append(model)
overrun = "fable" in model
return httpx.Response(
200,
json=CONTEXT_WINDOW_RESPONSE if overrun else OK_RESPONSE,
request=request,
)
def install(self, monkeypatch: MonkeyPatch) -> None:
async def _send(_client: httpx.AsyncClient, request: httpx.Request, **kwargs: Any) -> httpx.Response:
return await self.send(request, **kwargs)
monkeypatch.setattr(httpx.AsyncClient, "send", _send)
FABLE_TIER = {
"model_name": "fable-tier",
"litellm_params": {"model": "anthropic/claude-fable-5", "api_key": "sk-test"},
}
OPUS_TARGET = {
"model_name": "opus-target",
"litellm_params": {"model": "anthropic/claude-opus-5", "api_key": "sk-test"},
}
def _knob() -> dict[str, str]:
return {"model_context_window_exceeded": "RateLimitError"}
def _deployment_id(router: Router, index: int = 0) -> str:
return router.model_list[index]["model_info"]["id"]
@pytest.mark.asyncio
async def test_chat_completion_mapped_reason_falls_back_and_cools_down(monkeypatch: MonkeyPatch):
fake = FakeAnthropicUpstream()
router = Router(
model_list=[FABLE_TIER, OPUS_TARGET],
treat_finish_reason_as_failure=_knob(),
default_fallbacks=["opus-target"],
num_retries=0,
allowed_fails=0,
cooldown_time=10,
)
fake.install(monkeypatch)
response = await router.acompletion(model="fable-tier", max_tokens=16, messages=[{"role": "user", "content": "hi"}])
assert response.model == "claude-opus-5"
assert len(fake.calls) == 2
assert "claude-fable-5" in fake.calls[0]
assert "claude-opus-5" in fake.calls[1]
assert router.fail_calls["anthropic/claude-fable-5"] == 1
fable_id = _deployment_id(router, 0)
assert fable_id in _get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None)
@pytest.mark.asyncio
async def test_anthropic_messages_mapped_reason_falls_back(monkeypatch: MonkeyPatch):
fake = FakeAnthropicUpstream()
router = Router(
model_list=[FABLE_TIER, OPUS_TARGET],
treat_finish_reason_as_failure=_knob(),
default_fallbacks=["opus-target"],
num_retries=0,
allowed_fails=0,
cooldown_time=10,
)
fake.install(monkeypatch)
response = await router.aanthropic_messages(
model="fable-tier", max_tokens=16, messages=[{"role": "user", "content": "hi"}]
)
assert response["id"] == "msg_ok"
assert response["stop_reason"] == "end_turn"
assert len(fake.calls) == 2
assert "claude-opus-5" in fake.calls[1]
@pytest.mark.asyncio
async def test_mapped_reason_without_fallback_returns_response_and_counts_failure(monkeypatch: MonkeyPatch):
fake = FakeAnthropicUpstream()
router = Router(
model_list=[FABLE_TIER],
treat_finish_reason_as_failure=_knob(),
num_retries=0,
allowed_fails=0,
cooldown_time=10,
)
fake.install(monkeypatch)
response = await router.acompletion(model="fable-tier", max_tokens=16, messages=[{"role": "user", "content": "hi"}])
assert response.model == "claude-fable-5"
assert len(fake.calls) == 1
fable_id = _deployment_id(router, 0)
failures = router.cache.get_cache(local_only=True, key=f"{fable_id}:fails")
assert failures == 1
assert fable_id in _get_cooldown_deployments(litellm_router_instance=router, parent_otel_span=None)
@pytest.mark.asyncio
async def test_knob_unset_ignores_terminal_stop_reason(monkeypatch: MonkeyPatch):
fake = FakeAnthropicUpstream()
router = Router(model_list=[FABLE_TIER], num_retries=0)
fake.install(monkeypatch)
response = await router.acompletion(model="fable-tier", max_tokens=16, messages=[{"role": "user", "content": "hi"}])
assert response.model == "claude-fable-5"
assert len(fake.calls) == 1
assert router.fail_calls["fable-tier"] == 0
def test_unknown_exception_name_raises_at_construction():
with pytest.raises(ValueError, match="NotAnException"):
Router(model_list=[], treat_finish_reason_as_failure={"x": "NotAnException"})
def test_healthy_terminal_key_warns_at_construction(capsys: pytest.CaptureFixture):
Router(model_list=[], treat_finish_reason_as_failure={"stop": "RateLimitError"})
logged = capsys.readouterr().err + capsys.readouterr().out
assert "healthy terminal reasons" in logged
@pytest.mark.asyncio
async def test_mapped_finish_reason_helpers_direct(monkeypatch: MonkeyPatch):
"""Direct coverage of the knob helpers (the router code-coverage check matches by name)."""
fake = FakeAnthropicUpstream()
router = Router(
model_list=[FABLE_TIER, OPUS_TARGET],
treat_finish_reason_as_failure=_knob(),
default_fallbacks=["opus-target"],
num_retries=0,
allowed_fails=0,
cooldown_time=10,
)
fake.install(monkeypatch)
ok = await router.acompletion(model="opus-target", max_tokens=16, messages=[{"role": "user", "content": "hi"}])
assert router._get_mapped_finish_reason(ok) is None
assert router._generic_fallback_available("fable-tier", {}) is True
error = router._finish_reason_failure_error(model="fable-tier", reason="model_context_window_exceeded")
assert error.status_code == 429
deployment = router.model_list[0]
accounted = router._account_mapped_finish_reason_failure(
model="fable-tier",
deployment=deployment,
reason="model_context_window_exceeded",
kwargs={},
)
assert accounted is not None
with pytest.raises(litellm.RateLimitError):
router._handle_mapped_finish_reason_failure(
model="fable-tier",
deployment=deployment,
reason="model_context_window_exceeded",
kwargs={},
)