This commit is contained in:
武本龍門 2026-10-28 10:01:54 +09:00 • committed by GitHub
commit 5a1b5e3856
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 208 additions and 4 deletions

View file

@ -742,6 +742,56 @@ async def _is_fallback_target_within_budget(
return False
# Call-type dispatch stores the OCR entrypoint under one of these two kwargs keys depending on
# whether the request went through the plain or generic-API-call path (see
# references_provider_scoped_resource below for the same two-key pattern).
_OCR_HANDLER_FUNCTION_NAMES: Final = frozenset({"aocr", "ocr"})
# Every caller reaches run_async_fallback by spreading a loosely-typed kwargs dict, so this
# validates it once into a real Mapping[str, object] instead of threading the unannotated
# **kwargs type into each per-target check below.
_FALLBACK_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
def _is_ocr_fallback_request(kwargs: Mapping[str, object]) -> bool:
return any(
getattr(kwargs.get(function_key), "__name__", None) in _OCR_HANDLER_FUNCTION_NAMES
for function_key in ("original_function", "original_generic_function")
)
def _is_fallback_target_ocr_capable(
litellm_router: LitellmRouter,
fallback_entry: str | Mapping[str, object],
original_model_group: str,
kwargs: Mapping[str, object],
*,
attempted: AttemptedFallbackTargets,
) -> bool:
"""
An OCR call retried against a deployment explicitly configured for a different mode
(e.g. mode="chat") can never succeed: it costs a full round trip to that deployment
before failing the exact same way the primary deployment did. A model_group with no
mode configured is treated as compatible, since most deployments never set mode="ocr"
explicitly and an unset mode means "unknown", not "known incompatible".
"""
if not _is_ocr_fallback_request(kwargs):
return True
target: Final = _get_fallback_target_model_group(fallback_entry)
if target is None or target == original_model_group:
return True
configured_mode: Final = litellm_router.get_configured_mode(target)
if configured_mode is None or configured_mode == "ocr":
return True
verbose_router_logger.info(
"Skipping fallback to model_group = %s: configured mode = %s does not support OCR calls",
mask_sensitive_structure(fallback_entry),
configured_mode,
)
attempted.record_skip(target, FallbackSkipReason.MODE_INCOMPATIBLE)
return False
def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
"""
True when a file, batch, or fine-tuning job operation names an id that only exists
@ -830,6 +880,7 @@ async def run_async_fallback(
attempted: Final = attempted_fallback_targets_from(kwargs) # pyright: ignore[reportUnknownArgumentType] # keep legacy kwargs untyped
failed_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group
attempted.record(failed_model_group)
request_kwargs: Final[Mapping[str, object]] = _FALLBACK_REQUEST_KWARGS_ADAPTER.validate_python(kwargs, strict=True)
# A dict target is merged straight into kwargs below, and kwargs win over the deployment's own
# params, so a stored key/team/global fallback could otherwise set a federation field that the
@ -851,11 +902,15 @@ async def run_async_fallback(
)
continue
if not await _is_fallback_target_authorized(
litellm_router, mg, original_model_group, kwargs, attempted=attempted
litellm_router, mg, original_model_group, request_kwargs, attempted=attempted
):
continue
if not await _is_fallback_target_within_budget(
litellm_router, mg, original_model_group, kwargs, attempted=attempted
litellm_router, mg, original_model_group, request_kwargs, attempted=attempted
):
continue
if not _is_fallback_target_ocr_capable(
litellm_router, mg, original_model_group, request_kwargs, attempted=attempted
):
continue
attempt_key = fallback_attempt_key(mg)

View file

@ -69,6 +69,7 @@ AllowedModelRegion: TypeAlias = Literal["eu", "us"]
class FallbackSkipReason(str, enum.Enum):
NOT_AUTHORIZED = "not_authorized"
OVER_BUDGET = "over_budget"
MODE_INCOMPATIBLE = "mode_incompatible"
@dataclass(frozen=True, slots=True)
@ -972,7 +973,9 @@ class ModelGroupInfo(LiteLLMBaseModel):
input_cost_per_pixel: float | None = None
mode: (
str
| Literal["chat", "embedding", "completion", "image_generation", "audio_transcription", "rerank", "moderations"]
| Literal[
"chat", "embedding", "completion", "image_generation", "audio_transcription", "rerank", "moderations", "ocr"
]
| None
) = Field(default="chat")
tpm: int | None = None

View file

@ -27,6 +27,7 @@ from litellm.router_utils.fallback_event_handlers import (
PRE_ROUTING_SELECTED_MODEL_KEY,
AttemptedFallbackTargets,
MidStreamFallbackControls,
_is_fallback_target_ocr_capable,
_trigger_cooldown_for_failed_deployment,
attempted_retries_for_request,
carry_over_routed_deployment,
@ -793,6 +794,151 @@ async def test_run_async_fallback_does_not_consult_access_check_for_same_model_g
assert router.access_checks == []
async def aocr(*args: object, **kwargs: object) -> NoReturn:
raise AssertionError("only used for its __name__")
class OcrModeAwareRouter(AttemptRecordingRouter):
fallback_access_check = None
fallback_budget_check = None
def __init__(self, configured_modes: Dict[str, str]):
super().__init__()
self.configured_modes = configured_modes
def get_configured_mode(self, model_name):
return self.configured_modes.get(model_name)
@pytest.mark.asyncio
async def test_run_async_fallback_skips_ocr_incapable_target():
"""A chat-only fallback can never serve an OCR call, so retrying it only costs a
round trip before failing the same way the primary deployment did."""
router = OcrModeAwareRouter(configured_modes={"chat-only-model": "chat", "ocr-model": "ocr"})
await run_async_fallback(
litellm_router=router,
fallback_model_group=["chat-only-model", "ocr-model"],
original_model_group="primary-ocr-model",
original_exception=RuntimeError("primary ocr deployment failed"),
max_fallbacks=3,
fallback_depth=0,
original_function=aocr,
)
assert router.attempted_model_groups == ["ocr-model"]
@pytest.mark.asyncio
async def test_run_async_fallback_detects_ocr_via_generic_function_kwarg():
"""`/v1/ocr` dispatches through the generic-API-call path, which stores the real
handler under original_generic_function and overwrites original_function with the
generic helper (see references_provider_scoped_resource for the same two-key read)."""
router = OcrModeAwareRouter(configured_modes={"chat-only-model": "chat"})
with pytest.raises(RuntimeError, match="primary ocr deployment failed"):
await run_async_fallback(
litellm_router=router,
fallback_model_group=["chat-only-model"],
original_model_group="primary-ocr-model",
original_exception=RuntimeError("primary ocr deployment failed"),
max_fallbacks=3,
fallback_depth=0,
original_function=router.async_function_with_fallbacks,
original_generic_function=aocr,
)
assert router.attempted_model_groups == []
@pytest.mark.asyncio
async def test_run_async_fallback_treats_unconfigured_mode_as_ocr_capable():
"""Most deployments never set model_info.mode explicitly, so an unset mode reads as
"unknown", not "known incompatible" -- otherwise every ordinary OCR fallback would be
skipped by default."""
router = OcrModeAwareRouter(configured_modes={})
await run_async_fallback(
litellm_router=router,
fallback_model_group=["unconfigured-model"],
original_model_group="primary-ocr-model",
original_exception=RuntimeError("primary ocr deployment failed"),
max_fallbacks=3,
fallback_depth=0,
original_function=aocr,
)
assert router.attempted_model_groups == ["unconfigured-model"]
@pytest.mark.asyncio
async def test_run_async_fallback_does_not_mode_gate_non_ocr_calls():
"""The capability check is scoped to OCR calls only: a chat completion retried against
a model_group configured for a different mode is an existing, separate gap and must not
change behavior here."""
router = OcrModeAwareRouter(configured_modes={"ocr-only-model": "ocr"})
await run_async_fallback(
litellm_router=router,
fallback_model_group=["ocr-only-model"],
original_model_group="primary-model",
original_exception=RuntimeError("primary chat deployment failed"),
max_fallbacks=3,
fallback_depth=0,
original_function=_acompletion,
)
assert router.attempted_model_groups == ["ocr-only-model"]
@pytest.mark.asyncio
async def test_run_async_fallback_raises_original_error_when_no_ocr_capable_fallback_exists():
router = OcrModeAwareRouter(configured_modes={"chat-only-model": "chat", "other-chat-model": "chat"})
with pytest.raises(RuntimeError, match="primary ocr deployment failed"):
await run_async_fallback(
litellm_router=router,
fallback_model_group=["chat-only-model", "other-chat-model"],
original_model_group="primary-ocr-model",
original_exception=RuntimeError("primary ocr deployment failed"),
max_fallbacks=3,
fallback_depth=0,
original_function=aocr,
)
assert router.attempted_model_groups == []
def test_is_fallback_target_ocr_capable_treats_self_reference_as_compatible():
"""A fallback entry naming the same model_group as the original request is the same
deployment that already failed, not a different one being probed for capability, so it
is never mode-gated."""
router = OcrModeAwareRouter(configured_modes={"primary-ocr-model": "chat"})
assert _is_fallback_target_ocr_capable(
router,
"primary-ocr-model",
"primary-ocr-model",
{"original_function": aocr},
attempted=AttemptedFallbackTargets(),
)
def test_is_fallback_target_ocr_capable_treats_unresolvable_target_as_compatible():
"""A fallback entry with no resolvable model_group name (e.g. a dict missing `model`)
can't be looked up against a configured mode, so it is treated as compatible rather than
blocking a fallback the router would otherwise accept."""
router = OcrModeAwareRouter(configured_modes={})
assert _is_fallback_target_ocr_capable(
router,
{"not_model": "x"},
"primary-ocr-model",
{"original_function": aocr},
attempted=AttemptedFallbackTargets(),
)
class RecordingFailRouter:
fallback_access_check = None
fallback_budget_check = None

View file

@ -38461,7 +38461,7 @@ export interface components {
* Mode
* @default chat
*/
mode: string | ("chat" | "embedding" | "completion" | "image_generation" | "audio_transcription" | "rerank" | "moderations") | null;
mode: string | ("chat" | "embedding" | "completion" | "image_generation" | "audio_transcription" | "rerank" | "moderations" | "ocr") | null;
/** Model Group */
model_group: string;
/** Otpm */