From 0472e10de3cf384cde3c8fd1b42b8aa20dc3a070 Mon Sep 17 00:00:00 2001 From: "ryuto.takemoto" Date: Thu, 15 Oct 2026 11:55:46 +0900 Subject: [PATCH 1/3] feat(router): skip ocr-incapable fallback targets aocr/ocr fallback resolution matched purely on model_group name, so a fallback pointed at a chat-only deployment got retried every time and failed the same way the primary deployment did. Skip a candidate whose configured mode is known and incompatible with ocr; an unset mode is treated as unknown, not incompatible, since most deployments never set mode explicitly. Fixes #46564 --- .../router_utils/fallback_event_handlers.py | 44 +++++++ litellm/types/router.py | 4 +- .../test_fallback_event_handlers.py | 115 ++++++++++++++++++ 3 files changed, 162 insertions(+), 1 deletion(-) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index e1de57c91a0..36e9acb1c39 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -645,6 +645,48 @@ async def _is_fallback_target_within_budget( return False +# Call-type dispatch stores the OCR entrypoint under one of these two kwargs keys depending on +# whether the request went through the plain or generic-API-call path (see +# references_provider_scoped_resource below for the same two-key pattern). +_OCR_HANDLER_FUNCTION_NAMES: Final = frozenset({"aocr", "ocr"}) + + +def _is_ocr_fallback_request(kwargs: Mapping[str, object]) -> bool: + return any( + getattr(kwargs.get(function_key), "__name__", None) in _OCR_HANDLER_FUNCTION_NAMES + for function_key in ("original_function", "original_generic_function") + ) + + +def _is_fallback_target_ocr_capable( + litellm_router: LitellmRouter, + fallback_entry: str | Mapping[str, object], + original_model_group: str, + kwargs: Mapping[str, object], +) -> bool: + """ + An OCR call retried against a deployment explicitly configured for a different mode + (e.g. mode="chat") can never succeed: it costs a full round trip to that deployment + before failing the exact same way the primary deployment did. A model_group with no + mode configured is treated as compatible, since most deployments never set mode="ocr" + explicitly and an unset mode means "unknown", not "known incompatible". + """ + if not _is_ocr_fallback_request(kwargs): + return True + target: Final = _get_fallback_target_model_group(fallback_entry) + if target is None or target == original_model_group: + return True + configured_mode: Final = litellm_router.get_configured_mode(target) + if configured_mode is None or configured_mode == "ocr": + return True + verbose_router_logger.info( + "Skipping fallback to model_group = %s: configured mode = %s does not support OCR calls", + mask_sensitive_structure(fallback_entry), + configured_mode, + ) + return False + + def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool: """ True when a file, batch, or fine-tuning job operation names an id that only exists @@ -762,6 +804,8 @@ async def run_async_fallback( continue if not await _is_fallback_target_within_budget(litellm_router, mg, original_model_group, kwargs): continue + if not _is_fallback_target_ocr_capable(litellm_router, mg, original_model_group, kwargs): + continue attempt_key = fallback_attempt_key(mg) if attempt_key is not None: if attempt_key in attempted: diff --git a/litellm/types/router.py b/litellm/types/router.py index 273edf0f913..cca72e9d22f 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -909,7 +909,9 @@ class ModelGroupInfo(LiteLLMBaseModel): input_cost_per_pixel: float | None = None mode: ( str - | Literal["chat", "embedding", "completion", "image_generation", "audio_transcription", "rerank", "moderations"] + | Literal[ + "chat", "embedding", "completion", "image_generation", "audio_transcription", "rerank", "moderations", "ocr" + ] | None ) = Field(default="chat") tpm: int | None = None diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index febb33aab7d..1349df4630f 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -570,6 +570,121 @@ async def test_run_async_fallback_does_not_consult_access_check_for_same_model_g assert router.access_checks == [] +async def aocr(*args: object, **kwargs: object) -> NoReturn: + raise AssertionError("only used for its __name__") + + +class OcrModeAwareRouter(AttemptRecordingRouter): + fallback_access_check = None + fallback_budget_check = None + + def __init__(self, configured_modes: Dict[str, str]): + super().__init__() + self.configured_modes = configured_modes + + def get_configured_mode(self, model_name): + return self.configured_modes.get(model_name) + + +@pytest.mark.asyncio +async def test_run_async_fallback_skips_ocr_incapable_target(): + """A chat-only fallback can never serve an OCR call, so retrying it only costs a + round trip before failing the same way the primary deployment did.""" + router = OcrModeAwareRouter(configured_modes={"chat-only-model": "chat", "ocr-model": "ocr"}) + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["chat-only-model", "ocr-model"], + original_model_group="primary-ocr-model", + original_exception=RuntimeError("primary ocr deployment failed"), + max_fallbacks=3, + fallback_depth=0, + original_function=aocr, + ) + + assert router.attempted_model_groups == ["ocr-model"] + + +@pytest.mark.asyncio +async def test_run_async_fallback_detects_ocr_via_generic_function_kwarg(): + """`/v1/ocr` dispatches through the generic-API-call path, which stores the real + handler under original_generic_function and overwrites original_function with the + generic helper (see references_provider_scoped_resource for the same two-key read).""" + router = OcrModeAwareRouter(configured_modes={"chat-only-model": "chat"}) + + with pytest.raises(RuntimeError, match="primary ocr deployment failed"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=["chat-only-model"], + original_model_group="primary-ocr-model", + original_exception=RuntimeError("primary ocr deployment failed"), + max_fallbacks=3, + fallback_depth=0, + original_function=router.async_function_with_fallbacks, + original_generic_function=aocr, + ) + + assert router.attempted_model_groups == [] + + +@pytest.mark.asyncio +async def test_run_async_fallback_treats_unconfigured_mode_as_ocr_capable(): + """Most deployments never set model_info.mode explicitly, so an unset mode reads as + "unknown", not "known incompatible" -- otherwise every ordinary OCR fallback would be + skipped by default.""" + router = OcrModeAwareRouter(configured_modes={}) + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["unconfigured-model"], + original_model_group="primary-ocr-model", + original_exception=RuntimeError("primary ocr deployment failed"), + max_fallbacks=3, + fallback_depth=0, + original_function=aocr, + ) + + assert router.attempted_model_groups == ["unconfigured-model"] + + +@pytest.mark.asyncio +async def test_run_async_fallback_does_not_mode_gate_non_ocr_calls(): + """The capability check is scoped to OCR calls only: a chat completion retried against + a model_group configured for a different mode is an existing, separate gap and must not + change behavior here.""" + router = OcrModeAwareRouter(configured_modes={"ocr-only-model": "ocr"}) + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["ocr-only-model"], + original_model_group="primary-model", + original_exception=RuntimeError("primary chat deployment failed"), + max_fallbacks=3, + fallback_depth=0, + original_function=_acompletion, + ) + + assert router.attempted_model_groups == ["ocr-only-model"] + + +@pytest.mark.asyncio +async def test_run_async_fallback_raises_original_error_when_no_ocr_capable_fallback_exists(): + router = OcrModeAwareRouter(configured_modes={"chat-only-model": "chat", "other-chat-model": "chat"}) + + with pytest.raises(RuntimeError, match="primary ocr deployment failed"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=["chat-only-model", "other-chat-model"], + original_model_group="primary-ocr-model", + original_exception=RuntimeError("primary ocr deployment failed"), + max_fallbacks=3, + fallback_depth=0, + original_function=aocr, + ) + + assert router.attempted_model_groups == [] + + class RecordingFailRouter: fallback_access_check = None fallback_budget_check = None From 39b1ec37190524217726e2685ff67744bd5ce20c Mon Sep 17 00:00:00 2001 From: "ryuto.takemoto" Date: Mon, 19 Oct 2026 10:08:05 +0900 Subject: [PATCH 2/3] fix(router): keep fallback kwargs type-checked and regenerate ui types Validates kwargs once through a TypeAdapter before the per-target fallback checks instead of threading the unannotated **kwargs type into each one, which cleared a basedpyright ceiling breach. Also regenerates the dashboard's OpenAPI types for the ocr mode addition. --- litellm/router_utils/fallback_event_handlers.py | 14 +++++++++++--- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 36e9acb1c39..99bfa53a706 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -7,6 +7,8 @@ from enum import Enum from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final +from pydantic import TypeAdapter + import litellm from litellm._logging import verbose_router_logger from litellm.integrations.custom_logger import CustomLogger @@ -650,6 +652,11 @@ async def _is_fallback_target_within_budget( # references_provider_scoped_resource below for the same two-key pattern). _OCR_HANDLER_FUNCTION_NAMES: Final = frozenset({"aocr", "ocr"}) +# Every caller reaches run_async_fallback by spreading a loosely-typed kwargs dict, so this +# validates it once into a real Mapping[str, object] instead of threading the unannotated +# **kwargs type into each per-target check below. +_FALLBACK_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + def _is_ocr_fallback_request(kwargs: Mapping[str, object]) -> bool: return any( @@ -781,6 +788,7 @@ async def run_async_fallback( ) failed_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group attempted.record(failed_model_group) + request_kwargs: Final[Mapping[str, object]] = _FALLBACK_REQUEST_KWARGS_ADAPTER.validate_python(kwargs, strict=True) # A dict target is merged straight into kwargs below, and kwargs win over the deployment's own # params, so a stored key/team/global fallback could otherwise set a federation field that the @@ -800,11 +808,11 @@ async def run_async_fallback( original_model_group, ) continue - if not await _is_fallback_target_authorized(litellm_router, mg, original_model_group, kwargs): + if not await _is_fallback_target_authorized(litellm_router, mg, original_model_group, request_kwargs): continue - if not await _is_fallback_target_within_budget(litellm_router, mg, original_model_group, kwargs): + if not await _is_fallback_target_within_budget(litellm_router, mg, original_model_group, request_kwargs): continue - if not _is_fallback_target_ocr_capable(litellm_router, mg, original_model_group, kwargs): + if not _is_fallback_target_ocr_capable(litellm_router, mg, original_model_group, request_kwargs): continue attempt_key = fallback_attempt_key(mg) if attempt_key is not None: diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 98071b81e6f..ed1dd83861e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -37914,7 +37914,7 @@ export interface components { * Mode * @default chat */ - mode: string | ("chat" | "embedding" | "completion" | "image_generation" | "audio_transcription" | "rerank" | "moderations") | null; + mode: string | ("chat" | "embedding" | "completion" | "image_generation" | "audio_transcription" | "rerank" | "moderations" | "ocr") | null; /** Model Group */ model_group: string; /** Otpm */ From 85e3967a289d0ee35944155a404b02796628a8c6 Mon Sep 17 00:00:00 2001 From: "ryuto.takemoto" Date: Mon, 19 Oct 2026 16:05:01 +0900 Subject: [PATCH 3/3] test(router): cover the ocr-capability check's self-reference and unresolvable-target branches codecov flagged the "target is None or target == original_model_group" line in _is_fallback_target_ocr_capable as only partially covered. --- .../test_fallback_event_handlers.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index f7fe480e8ea..c3a285a7fa0 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -25,6 +25,7 @@ from litellm.router_utils.fallback_event_handlers import ( PRE_ROUTING_SELECTED_MODEL_KEY, AttemptedFallbackTargets, MidStreamFallbackControls, + _is_fallback_target_ocr_capable, _trigger_cooldown_for_failed_deployment, attempted_retries_for_request, carry_over_routed_deployment, @@ -692,6 +693,36 @@ async def test_run_async_fallback_raises_original_error_when_no_ocr_capable_fall assert router.attempted_model_groups == [] +def test_is_fallback_target_ocr_capable_treats_self_reference_as_compatible(): + """A fallback entry naming the same model_group as the original request is the same + deployment that already failed, not a different one being probed for capability, so it + is never mode-gated.""" + router = OcrModeAwareRouter(configured_modes={"primary-ocr-model": "chat"}) + + assert _is_fallback_target_ocr_capable( + router, + "primary-ocr-model", + "primary-ocr-model", + {"original_function": aocr}, + attempted=AttemptedFallbackTargets(), + ) + + +def test_is_fallback_target_ocr_capable_treats_unresolvable_target_as_compatible(): + """A fallback entry with no resolvable model_group name (e.g. a dict missing `model`) + can't be looked up against a configured mode, so it is treated as compatible rather than + blocking a fallback the router would otherwise accept.""" + router = OcrModeAwareRouter(configured_modes={}) + + assert _is_fallback_target_ocr_capable( + router, + {"not_model": "x"}, + "primary-ocr-model", + {"original_function": aocr}, + attempted=AttemptedFallbackTargets(), + ) + + class RecordingFailRouter: fallback_access_check = None fallback_budget_check = None