diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index 6c93fe6d239..2d63e0038ef 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -742,6 +742,56 @@ async def _is_fallback_target_within_budget( return False +# Call-type dispatch stores the OCR entrypoint under one of these two kwargs keys depending on +# whether the request went through the plain or generic-API-call path (see +# references_provider_scoped_resource below for the same two-key pattern). +_OCR_HANDLER_FUNCTION_NAMES: Final = frozenset({"aocr", "ocr"}) + +# Every caller reaches run_async_fallback by spreading a loosely-typed kwargs dict, so this +# validates it once into a real Mapping[str, object] instead of threading the unannotated +# **kwargs type into each per-target check below. +_FALLBACK_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object]) + + +def _is_ocr_fallback_request(kwargs: Mapping[str, object]) -> bool: + return any( + getattr(kwargs.get(function_key), "__name__", None) in _OCR_HANDLER_FUNCTION_NAMES + for function_key in ("original_function", "original_generic_function") + ) + + +def _is_fallback_target_ocr_capable( + litellm_router: LitellmRouter, + fallback_entry: str | Mapping[str, object], + original_model_group: str, + kwargs: Mapping[str, object], + *, + attempted: AttemptedFallbackTargets, +) -> bool: + """ + An OCR call retried against a deployment explicitly configured for a different mode + (e.g. mode="chat") can never succeed: it costs a full round trip to that deployment + before failing the exact same way the primary deployment did. A model_group with no + mode configured is treated as compatible, since most deployments never set mode="ocr" + explicitly and an unset mode means "unknown", not "known incompatible". + """ + if not _is_ocr_fallback_request(kwargs): + return True + target: Final = _get_fallback_target_model_group(fallback_entry) + if target is None or target == original_model_group: + return True + configured_mode: Final = litellm_router.get_configured_mode(target) + if configured_mode is None or configured_mode == "ocr": + return True + verbose_router_logger.info( + "Skipping fallback to model_group = %s: configured mode = %s does not support OCR calls", + mask_sensitive_structure(fallback_entry), + configured_mode, + ) + attempted.record_skip(target, FallbackSkipReason.MODE_INCOMPATIBLE) + return False + + def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool: """ True when a file, batch, or fine-tuning job operation names an id that only exists @@ -830,6 +880,7 @@ async def run_async_fallback( attempted: Final = attempted_fallback_targets_from(kwargs) # pyright: ignore[reportUnknownArgumentType] # keep legacy kwargs untyped failed_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group attempted.record(failed_model_group) + request_kwargs: Final[Mapping[str, object]] = _FALLBACK_REQUEST_KWARGS_ADAPTER.validate_python(kwargs, strict=True) # A dict target is merged straight into kwargs below, and kwargs win over the deployment's own # params, so a stored key/team/global fallback could otherwise set a federation field that the @@ -851,11 +902,15 @@ async def run_async_fallback( ) continue if not await _is_fallback_target_authorized( - litellm_router, mg, original_model_group, kwargs, attempted=attempted + litellm_router, mg, original_model_group, request_kwargs, attempted=attempted ): continue if not await _is_fallback_target_within_budget( - litellm_router, mg, original_model_group, kwargs, attempted=attempted + litellm_router, mg, original_model_group, request_kwargs, attempted=attempted + ): + continue + if not _is_fallback_target_ocr_capable( + litellm_router, mg, original_model_group, request_kwargs, attempted=attempted ): continue attempt_key = fallback_attempt_key(mg) diff --git a/litellm/types/router.py b/litellm/types/router.py index ba9b82e2a57..dcc5351e00c 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -69,6 +69,7 @@ AllowedModelRegion: TypeAlias = Literal["eu", "us"] class FallbackSkipReason(str, enum.Enum): NOT_AUTHORIZED = "not_authorized" OVER_BUDGET = "over_budget" + MODE_INCOMPATIBLE = "mode_incompatible" @dataclass(frozen=True, slots=True) @@ -972,7 +973,9 @@ class ModelGroupInfo(LiteLLMBaseModel): input_cost_per_pixel: float | None = None mode: ( str - | Literal["chat", "embedding", "completion", "image_generation", "audio_transcription", "rerank", "moderations"] + | Literal[ + "chat", "embedding", "completion", "image_generation", "audio_transcription", "rerank", "moderations", "ocr" + ] | None ) = Field(default="chat") tpm: int | None = None diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index c5a476c450b..b33933eb7df 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -27,6 +27,7 @@ from litellm.router_utils.fallback_event_handlers import ( PRE_ROUTING_SELECTED_MODEL_KEY, AttemptedFallbackTargets, MidStreamFallbackControls, + _is_fallback_target_ocr_capable, _trigger_cooldown_for_failed_deployment, attempted_retries_for_request, carry_over_routed_deployment, @@ -793,6 +794,151 @@ async def test_run_async_fallback_does_not_consult_access_check_for_same_model_g assert router.access_checks == [] +async def aocr(*args: object, **kwargs: object) -> NoReturn: + raise AssertionError("only used for its __name__") + + +class OcrModeAwareRouter(AttemptRecordingRouter): + fallback_access_check = None + fallback_budget_check = None + + def __init__(self, configured_modes: Dict[str, str]): + super().__init__() + self.configured_modes = configured_modes + + def get_configured_mode(self, model_name): + return self.configured_modes.get(model_name) + + +@pytest.mark.asyncio +async def test_run_async_fallback_skips_ocr_incapable_target(): + """A chat-only fallback can never serve an OCR call, so retrying it only costs a + round trip before failing the same way the primary deployment did.""" + router = OcrModeAwareRouter(configured_modes={"chat-only-model": "chat", "ocr-model": "ocr"}) + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["chat-only-model", "ocr-model"], + original_model_group="primary-ocr-model", + original_exception=RuntimeError("primary ocr deployment failed"), + max_fallbacks=3, + fallback_depth=0, + original_function=aocr, + ) + + assert router.attempted_model_groups == ["ocr-model"] + + +@pytest.mark.asyncio +async def test_run_async_fallback_detects_ocr_via_generic_function_kwarg(): + """`/v1/ocr` dispatches through the generic-API-call path, which stores the real + handler under original_generic_function and overwrites original_function with the + generic helper (see references_provider_scoped_resource for the same two-key read).""" + router = OcrModeAwareRouter(configured_modes={"chat-only-model": "chat"}) + + with pytest.raises(RuntimeError, match="primary ocr deployment failed"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=["chat-only-model"], + original_model_group="primary-ocr-model", + original_exception=RuntimeError("primary ocr deployment failed"), + max_fallbacks=3, + fallback_depth=0, + original_function=router.async_function_with_fallbacks, + original_generic_function=aocr, + ) + + assert router.attempted_model_groups == [] + + +@pytest.mark.asyncio +async def test_run_async_fallback_treats_unconfigured_mode_as_ocr_capable(): + """Most deployments never set model_info.mode explicitly, so an unset mode reads as + "unknown", not "known incompatible" -- otherwise every ordinary OCR fallback would be + skipped by default.""" + router = OcrModeAwareRouter(configured_modes={}) + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["unconfigured-model"], + original_model_group="primary-ocr-model", + original_exception=RuntimeError("primary ocr deployment failed"), + max_fallbacks=3, + fallback_depth=0, + original_function=aocr, + ) + + assert router.attempted_model_groups == ["unconfigured-model"] + + +@pytest.mark.asyncio +async def test_run_async_fallback_does_not_mode_gate_non_ocr_calls(): + """The capability check is scoped to OCR calls only: a chat completion retried against + a model_group configured for a different mode is an existing, separate gap and must not + change behavior here.""" + router = OcrModeAwareRouter(configured_modes={"ocr-only-model": "ocr"}) + + await run_async_fallback( + litellm_router=router, + fallback_model_group=["ocr-only-model"], + original_model_group="primary-model", + original_exception=RuntimeError("primary chat deployment failed"), + max_fallbacks=3, + fallback_depth=0, + original_function=_acompletion, + ) + + assert router.attempted_model_groups == ["ocr-only-model"] + + +@pytest.mark.asyncio +async def test_run_async_fallback_raises_original_error_when_no_ocr_capable_fallback_exists(): + router = OcrModeAwareRouter(configured_modes={"chat-only-model": "chat", "other-chat-model": "chat"}) + + with pytest.raises(RuntimeError, match="primary ocr deployment failed"): + await run_async_fallback( + litellm_router=router, + fallback_model_group=["chat-only-model", "other-chat-model"], + original_model_group="primary-ocr-model", + original_exception=RuntimeError("primary ocr deployment failed"), + max_fallbacks=3, + fallback_depth=0, + original_function=aocr, + ) + + assert router.attempted_model_groups == [] + + +def test_is_fallback_target_ocr_capable_treats_self_reference_as_compatible(): + """A fallback entry naming the same model_group as the original request is the same + deployment that already failed, not a different one being probed for capability, so it + is never mode-gated.""" + router = OcrModeAwareRouter(configured_modes={"primary-ocr-model": "chat"}) + + assert _is_fallback_target_ocr_capable( + router, + "primary-ocr-model", + "primary-ocr-model", + {"original_function": aocr}, + attempted=AttemptedFallbackTargets(), + ) + + +def test_is_fallback_target_ocr_capable_treats_unresolvable_target_as_compatible(): + """A fallback entry with no resolvable model_group name (e.g. a dict missing `model`) + can't be looked up against a configured mode, so it is treated as compatible rather than + blocking a fallback the router would otherwise accept.""" + router = OcrModeAwareRouter(configured_modes={}) + + assert _is_fallback_target_ocr_capable( + router, + {"not_model": "x"}, + "primary-ocr-model", + {"original_function": aocr}, + attempted=AttemptedFallbackTargets(), + ) + + class RecordingFailRouter: fallback_access_check = None fallback_budget_check = None diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 13a8a8b993d..c6b026e2f87 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -38461,7 +38461,7 @@ export interface components { * Mode * @default chat */ - mode: string | ("chat" | "embedding" | "completion" | "image_generation" | "audio_transcription" | "rerank" | "moderations") | null; + mode: string | ("chat" | "embedding" | "completion" | "image_generation" | "audio_transcription" | "rerank" | "moderations" | "ocr") | null; /** Model Group */ model_group: string; /** Otpm */