mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
Merge 85e3967a28 into e5fe716d69
This commit is contained in:
commit
5a1b5e3856
4 changed files with 208 additions and 4 deletions
|
|
@ -742,6 +742,56 @@ async def _is_fallback_target_within_budget(
|
|||
return False
|
||||
|
||||
|
||||
# Call-type dispatch stores the OCR entrypoint under one of these two kwargs keys depending on
|
||||
# whether the request went through the plain or generic-API-call path (see
|
||||
# references_provider_scoped_resource below for the same two-key pattern).
|
||||
_OCR_HANDLER_FUNCTION_NAMES: Final = frozenset({"aocr", "ocr"})
|
||||
|
||||
# Every caller reaches run_async_fallback by spreading a loosely-typed kwargs dict, so this
|
||||
# validates it once into a real Mapping[str, object] instead of threading the unannotated
|
||||
# **kwargs type into each per-target check below.
|
||||
_FALLBACK_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _is_ocr_fallback_request(kwargs: Mapping[str, object]) -> bool:
|
||||
return any(
|
||||
getattr(kwargs.get(function_key), "__name__", None) in _OCR_HANDLER_FUNCTION_NAMES
|
||||
for function_key in ("original_function", "original_generic_function")
|
||||
)
|
||||
|
||||
|
||||
def _is_fallback_target_ocr_capable(
|
||||
litellm_router: LitellmRouter,
|
||||
fallback_entry: str | Mapping[str, object],
|
||||
original_model_group: str,
|
||||
kwargs: Mapping[str, object],
|
||||
*,
|
||||
attempted: AttemptedFallbackTargets,
|
||||
) -> bool:
|
||||
"""
|
||||
An OCR call retried against a deployment explicitly configured for a different mode
|
||||
(e.g. mode="chat") can never succeed: it costs a full round trip to that deployment
|
||||
before failing the exact same way the primary deployment did. A model_group with no
|
||||
mode configured is treated as compatible, since most deployments never set mode="ocr"
|
||||
explicitly and an unset mode means "unknown", not "known incompatible".
|
||||
"""
|
||||
if not _is_ocr_fallback_request(kwargs):
|
||||
return True
|
||||
target: Final = _get_fallback_target_model_group(fallback_entry)
|
||||
if target is None or target == original_model_group:
|
||||
return True
|
||||
configured_mode: Final = litellm_router.get_configured_mode(target)
|
||||
if configured_mode is None or configured_mode == "ocr":
|
||||
return True
|
||||
verbose_router_logger.info(
|
||||
"Skipping fallback to model_group = %s: configured mode = %s does not support OCR calls",
|
||||
mask_sensitive_structure(fallback_entry),
|
||||
configured_mode,
|
||||
)
|
||||
attempted.record_skip(target, FallbackSkipReason.MODE_INCOMPATIBLE)
|
||||
return False
|
||||
|
||||
|
||||
def references_provider_scoped_resource(kwargs: Mapping[str, object]) -> bool:
|
||||
"""
|
||||
True when a file, batch, or fine-tuning job operation names an id that only exists
|
||||
|
|
@ -830,6 +880,7 @@ async def run_async_fallback(
|
|||
attempted: Final = attempted_fallback_targets_from(kwargs) # pyright: ignore[reportUnknownArgumentType] # keep legacy kwargs untyped
|
||||
failed_model_group: Final = get_pre_routing_selection(kwargs) or original_model_group
|
||||
attempted.record(failed_model_group)
|
||||
request_kwargs: Final[Mapping[str, object]] = _FALLBACK_REQUEST_KWARGS_ADAPTER.validate_python(kwargs, strict=True)
|
||||
|
||||
# A dict target is merged straight into kwargs below, and kwargs win over the deployment's own
|
||||
# params, so a stored key/team/global fallback could otherwise set a federation field that the
|
||||
|
|
@ -851,11 +902,15 @@ async def run_async_fallback(
|
|||
)
|
||||
continue
|
||||
if not await _is_fallback_target_authorized(
|
||||
litellm_router, mg, original_model_group, kwargs, attempted=attempted
|
||||
litellm_router, mg, original_model_group, request_kwargs, attempted=attempted
|
||||
):
|
||||
continue
|
||||
if not await _is_fallback_target_within_budget(
|
||||
litellm_router, mg, original_model_group, kwargs, attempted=attempted
|
||||
litellm_router, mg, original_model_group, request_kwargs, attempted=attempted
|
||||
):
|
||||
continue
|
||||
if not _is_fallback_target_ocr_capable(
|
||||
litellm_router, mg, original_model_group, request_kwargs, attempted=attempted
|
||||
):
|
||||
continue
|
||||
attempt_key = fallback_attempt_key(mg)
|
||||
|
|
|
|||
|
|
@ -69,6 +69,7 @@ AllowedModelRegion: TypeAlias = Literal["eu", "us"]
|
|||
class FallbackSkipReason(str, enum.Enum):
|
||||
NOT_AUTHORIZED = "not_authorized"
|
||||
OVER_BUDGET = "over_budget"
|
||||
MODE_INCOMPATIBLE = "mode_incompatible"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -972,7 +973,9 @@ class ModelGroupInfo(LiteLLMBaseModel):
|
|||
input_cost_per_pixel: float | None = None
|
||||
mode: (
|
||||
str
|
||||
| Literal["chat", "embedding", "completion", "image_generation", "audio_transcription", "rerank", "moderations"]
|
||||
| Literal[
|
||||
"chat", "embedding", "completion", "image_generation", "audio_transcription", "rerank", "moderations", "ocr"
|
||||
]
|
||||
| None
|
||||
) = Field(default="chat")
|
||||
tpm: int | None = None
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from litellm.router_utils.fallback_event_handlers import (
|
|||
PRE_ROUTING_SELECTED_MODEL_KEY,
|
||||
AttemptedFallbackTargets,
|
||||
MidStreamFallbackControls,
|
||||
_is_fallback_target_ocr_capable,
|
||||
_trigger_cooldown_for_failed_deployment,
|
||||
attempted_retries_for_request,
|
||||
carry_over_routed_deployment,
|
||||
|
|
@ -793,6 +794,151 @@ async def test_run_async_fallback_does_not_consult_access_check_for_same_model_g
|
|||
assert router.access_checks == []
|
||||
|
||||
|
||||
async def aocr(*args: object, **kwargs: object) -> NoReturn:
|
||||
raise AssertionError("only used for its __name__")
|
||||
|
||||
|
||||
class OcrModeAwareRouter(AttemptRecordingRouter):
|
||||
fallback_access_check = None
|
||||
fallback_budget_check = None
|
||||
|
||||
def __init__(self, configured_modes: Dict[str, str]):
|
||||
super().__init__()
|
||||
self.configured_modes = configured_modes
|
||||
|
||||
def get_configured_mode(self, model_name):
|
||||
return self.configured_modes.get(model_name)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_fallback_skips_ocr_incapable_target():
|
||||
"""A chat-only fallback can never serve an OCR call, so retrying it only costs a
|
||||
round trip before failing the same way the primary deployment did."""
|
||||
router = OcrModeAwareRouter(configured_modes={"chat-only-model": "chat", "ocr-model": "ocr"})
|
||||
|
||||
await run_async_fallback(
|
||||
litellm_router=router,
|
||||
fallback_model_group=["chat-only-model", "ocr-model"],
|
||||
original_model_group="primary-ocr-model",
|
||||
original_exception=RuntimeError("primary ocr deployment failed"),
|
||||
max_fallbacks=3,
|
||||
fallback_depth=0,
|
||||
original_function=aocr,
|
||||
)
|
||||
|
||||
assert router.attempted_model_groups == ["ocr-model"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_fallback_detects_ocr_via_generic_function_kwarg():
|
||||
"""`/v1/ocr` dispatches through the generic-API-call path, which stores the real
|
||||
handler under original_generic_function and overwrites original_function with the
|
||||
generic helper (see references_provider_scoped_resource for the same two-key read)."""
|
||||
router = OcrModeAwareRouter(configured_modes={"chat-only-model": "chat"})
|
||||
|
||||
with pytest.raises(RuntimeError, match="primary ocr deployment failed"):
|
||||
await run_async_fallback(
|
||||
litellm_router=router,
|
||||
fallback_model_group=["chat-only-model"],
|
||||
original_model_group="primary-ocr-model",
|
||||
original_exception=RuntimeError("primary ocr deployment failed"),
|
||||
max_fallbacks=3,
|
||||
fallback_depth=0,
|
||||
original_function=router.async_function_with_fallbacks,
|
||||
original_generic_function=aocr,
|
||||
)
|
||||
|
||||
assert router.attempted_model_groups == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_fallback_treats_unconfigured_mode_as_ocr_capable():
|
||||
"""Most deployments never set model_info.mode explicitly, so an unset mode reads as
|
||||
"unknown", not "known incompatible" -- otherwise every ordinary OCR fallback would be
|
||||
skipped by default."""
|
||||
router = OcrModeAwareRouter(configured_modes={})
|
||||
|
||||
await run_async_fallback(
|
||||
litellm_router=router,
|
||||
fallback_model_group=["unconfigured-model"],
|
||||
original_model_group="primary-ocr-model",
|
||||
original_exception=RuntimeError("primary ocr deployment failed"),
|
||||
max_fallbacks=3,
|
||||
fallback_depth=0,
|
||||
original_function=aocr,
|
||||
)
|
||||
|
||||
assert router.attempted_model_groups == ["unconfigured-model"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_fallback_does_not_mode_gate_non_ocr_calls():
|
||||
"""The capability check is scoped to OCR calls only: a chat completion retried against
|
||||
a model_group configured for a different mode is an existing, separate gap and must not
|
||||
change behavior here."""
|
||||
router = OcrModeAwareRouter(configured_modes={"ocr-only-model": "ocr"})
|
||||
|
||||
await run_async_fallback(
|
||||
litellm_router=router,
|
||||
fallback_model_group=["ocr-only-model"],
|
||||
original_model_group="primary-model",
|
||||
original_exception=RuntimeError("primary chat deployment failed"),
|
||||
max_fallbacks=3,
|
||||
fallback_depth=0,
|
||||
original_function=_acompletion,
|
||||
)
|
||||
|
||||
assert router.attempted_model_groups == ["ocr-only-model"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_async_fallback_raises_original_error_when_no_ocr_capable_fallback_exists():
|
||||
router = OcrModeAwareRouter(configured_modes={"chat-only-model": "chat", "other-chat-model": "chat"})
|
||||
|
||||
with pytest.raises(RuntimeError, match="primary ocr deployment failed"):
|
||||
await run_async_fallback(
|
||||
litellm_router=router,
|
||||
fallback_model_group=["chat-only-model", "other-chat-model"],
|
||||
original_model_group="primary-ocr-model",
|
||||
original_exception=RuntimeError("primary ocr deployment failed"),
|
||||
max_fallbacks=3,
|
||||
fallback_depth=0,
|
||||
original_function=aocr,
|
||||
)
|
||||
|
||||
assert router.attempted_model_groups == []
|
||||
|
||||
|
||||
def test_is_fallback_target_ocr_capable_treats_self_reference_as_compatible():
|
||||
"""A fallback entry naming the same model_group as the original request is the same
|
||||
deployment that already failed, not a different one being probed for capability, so it
|
||||
is never mode-gated."""
|
||||
router = OcrModeAwareRouter(configured_modes={"primary-ocr-model": "chat"})
|
||||
|
||||
assert _is_fallback_target_ocr_capable(
|
||||
router,
|
||||
"primary-ocr-model",
|
||||
"primary-ocr-model",
|
||||
{"original_function": aocr},
|
||||
attempted=AttemptedFallbackTargets(),
|
||||
)
|
||||
|
||||
|
||||
def test_is_fallback_target_ocr_capable_treats_unresolvable_target_as_compatible():
|
||||
"""A fallback entry with no resolvable model_group name (e.g. a dict missing `model`)
|
||||
can't be looked up against a configured mode, so it is treated as compatible rather than
|
||||
blocking a fallback the router would otherwise accept."""
|
||||
router = OcrModeAwareRouter(configured_modes={})
|
||||
|
||||
assert _is_fallback_target_ocr_capable(
|
||||
router,
|
||||
{"not_model": "x"},
|
||||
"primary-ocr-model",
|
||||
{"original_function": aocr},
|
||||
attempted=AttemptedFallbackTargets(),
|
||||
)
|
||||
|
||||
|
||||
class RecordingFailRouter:
|
||||
fallback_access_check = None
|
||||
fallback_budget_check = None
|
||||
|
|
|
|||
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
2
ui/litellm-dashboard/src/lib/http/schema.d.ts
generated
vendored
|
|
@ -38461,7 +38461,7 @@ export interface components {
|
|||
* Mode
|
||||
* @default chat
|
||||
*/
|
||||
mode: string | ("chat" | "embedding" | "completion" | "image_generation" | "audio_transcription" | "rerank" | "moderations") | null;
|
||||
mode: string | ("chat" | "embedding" | "completion" | "image_generation" | "audio_transcription" | "rerank" | "moderations" | "ocr") | null;
|
||||
/** Model Group */
|
||||
model_group: string;
|
||||
/** Otpm */
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue