mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-27 06:13:07 +00:00
fix(router): skip declared text-only deployments for image requests on every routing strategy (#48241)
* fix(router): skip declared text-only deployments for image requests without enable_pre_call_checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): run the vision filter after the cooldown filter so a cooled vision leg falls back to the text leg Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(router): apply the vision filter after pre-call checks and tidy the new tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(router): mark the remaining locals in the vision filter tests Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: joshua <joshua@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
1418c1990f
commit
34ec3fcdfa
4 changed files with 135 additions and 46 deletions
|
|
@ -4821,10 +4821,15 @@ class ProxyBaseLLMRequestProcessing:
|
|||
model_info = metadata.get("model_info") or {}
|
||||
model_id = model_info.get("id", None)
|
||||
|
||||
# 3. Final fallback to self.data["litellm_metadata"] (for routes like /v1/responses that populate data before error)
|
||||
# 3. Final fallback to the request metadata the router updates in place with the selected deployment
|
||||
if not model_id:
|
||||
litellm_metadata: Final = self.data.get("litellm_metadata", {}) or {}
|
||||
model_info = litellm_metadata.get("model_info", {}) or {}
|
||||
model_id = model_info.get("id", None)
|
||||
model_id = self._metadata_model_id("litellm_metadata") or self._metadata_model_id("metadata")
|
||||
|
||||
return model_id
|
||||
|
||||
def _metadata_model_id(self, metadata_key: str) -> str | None:
|
||||
match self.data.get(metadata_key):
|
||||
case {"model_info": {"id": str() as request_model_id}}:
|
||||
return request_model_id
|
||||
case _:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -14315,6 +14315,30 @@ class Router:
|
|||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, filtered_deployments) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _without_declared_text_only_deployments(
|
||||
healthy_deployments: _RouteDeploymentsT,
|
||||
group: str,
|
||||
messages: list[dict[str, str]] | None, # mutable-ok: handed to _request_carries_image as-is
|
||||
request_kwargs: Mapping[str, object] | None,
|
||||
) -> _RouteDeploymentsT:
|
||||
if isinstance(healthy_deployments, Mapping) or not Router._request_carries_image(
|
||||
messages, request_kwargs or {}
|
||||
):
|
||||
return healthy_deployments
|
||||
|
||||
vision_capable_deployments: Final = [
|
||||
deployment
|
||||
for deployment in healthy_deployments
|
||||
if not Router._deployment_declares_no_vision(
|
||||
cast(DeploymentTypedDict, deployment), # cast-ok: router candidates use the deployment shape
|
||||
group,
|
||||
)
|
||||
]
|
||||
if not vision_capable_deployments:
|
||||
return healthy_deployments
|
||||
return cast(_RouteDeploymentsT, vision_capable_deployments) # cast-ok: same list shape as the input candidates
|
||||
|
||||
@staticmethod
|
||||
def _request_carries_image(
|
||||
messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is
|
||||
|
|
@ -14355,7 +14379,6 @@ class Router:
|
|||
- model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
|
||||
- filter models above rpm limits
|
||||
- if region given, filter out models not in that region / unknown region
|
||||
- request carries an image and the deployment declares supports_vision false, unless every candidate does
|
||||
- [TODO] function call and model doesn't support function calling
|
||||
"""
|
||||
|
||||
|
|
@ -14515,20 +14538,9 @@ class Router:
|
|||
model=model,
|
||||
llm_provider="",
|
||||
)
|
||||
remaining: Final = tuple(
|
||||
(idx, deployment) for idx, deployment in param_candidates if idx not in excluded_indices
|
||||
)
|
||||
text_only_indices: Final[frozenset[int]] = (
|
||||
frozenset(idx for idx, deployment in remaining if self._deployment_declares_no_vision(deployment, model))
|
||||
if remaining and self._request_carries_image(messages, request_settings)
|
||||
else frozenset()
|
||||
)
|
||||
dropped_indices: Final[frozenset[int]] = excluded_indices | (
|
||||
text_only_indices if len(text_only_indices) < len(remaining) else frozenset()
|
||||
)
|
||||
if len(dropped_indices) > 0:
|
||||
if len(excluded_indices) > 0:
|
||||
# Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops)
|
||||
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in dropped_indices]
|
||||
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices]
|
||||
|
||||
return _returned_deployments
|
||||
|
||||
|
|
@ -15051,8 +15063,11 @@ class Router:
|
|||
|
||||
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
|
||||
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
|
||||
batch_candidates: Final = cast( # cast-ok: router deployments are plain dicts
|
||||
list[dict[str, object]], healthy_deployments
|
||||
batch_candidates: Final = self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
)
|
||||
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
|
||||
batch_candidates,
|
||||
|
|
@ -16291,8 +16306,11 @@ class Router:
|
|||
|
||||
## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
|
||||
_target_order: Final = (request_kwargs or {}).pop("_target_order", None)
|
||||
batch_candidates: Final = cast( # cast-ok: router deployments are plain dicts
|
||||
list[dict[str, object]], healthy_deployments
|
||||
batch_candidates: Final = self._without_declared_text_only_deployments(
|
||||
cast(list[dict[str, object]], healthy_deployments), # cast-ok: router deployments are plain dicts
|
||||
model,
|
||||
messages,
|
||||
cast(Mapping[str, object] | None, request_kwargs), # cast-ok: legacy signature uses a bare dict
|
||||
)
|
||||
embedding_batch_admitting: Final = deployments_admitting_embedding_batch(
|
||||
batch_candidates,
|
||||
|
|
|
|||
|
|
@ -12027,3 +12027,24 @@ class TestResponseCostFromLoggingObjForCacheHits:
|
|||
)
|
||||
|
||||
assert cost == ""
|
||||
|
||||
|
||||
def test_maybe_get_model_id_reads_the_chat_metadata_the_router_updated_in_place():
|
||||
processing_obj: Final = ProxyBaseLLMRequestProcessing(data={"metadata": {"model_info": {"id": "declared-text"}}})
|
||||
|
||||
assert processing_obj.maybe_get_model_id(None) == "declared-text"
|
||||
|
||||
|
||||
def test_maybe_get_model_id_prefers_litellm_metadata_over_caller_metadata_without_a_deployment():
|
||||
processing_obj: Final = ProxyBaseLLMRequestProcessing(
|
||||
data={
|
||||
"metadata": {"trace_id": "t-1"},
|
||||
"litellm_metadata": {"model_info": {"id": "vision"}},
|
||||
}
|
||||
)
|
||||
|
||||
assert processing_obj.maybe_get_model_id(None) == "vision"
|
||||
|
||||
|
||||
def test_maybe_get_model_id_returns_none_before_the_router_picked_a_deployment():
|
||||
assert ProxyBaseLLMRequestProcessing(data={"metadata": {}}).maybe_get_model_id(None) is None
|
||||
|
|
|
|||
|
|
@ -30022,64 +30022,109 @@ _DECLARED_TEXT_LEG: Final[dict[str, object]] = {
|
|||
}
|
||||
|
||||
|
||||
def _pre_call_router(*deployments: dict[str, object], routing_strategy: str = "latency-based-routing") -> litellm.Router:
|
||||
def _mixed_router(*deployments: dict[str, object], routing_strategy: str = "latency-based-routing") -> litellm.Router:
|
||||
return litellm.Router(
|
||||
model_list=[{"model_name": "mixed", **deployment} for deployment in deployments],
|
||||
routing_strategy=routing_strategy,
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
|
||||
|
||||
def _pre_call_ids(router: litellm.Router, deployments: tuple[dict[str, object], ...], **request: object) -> list[str]:
|
||||
return [d["model_info"]["id"] for d in router._pre_call_checks(model="mixed", healthy_deployments=list(deployments), **request)]
|
||||
def _candidate_ids(
|
||||
router: litellm.Router,
|
||||
messages: list[dict[str, object]] | None = None,
|
||||
request_kwargs: dict[str, object] | None = None,
|
||||
) -> list[str]:
|
||||
candidates: Final = router._without_declared_text_only_deployments(
|
||||
router.get_model_list(model_name="mixed"), "mixed", messages, request_kwargs
|
||||
)
|
||||
return [d["model_info"]["id"] for d in candidates]
|
||||
|
||||
|
||||
def test_pre_call_checks_drop_declared_text_only_legs_for_image_requests():
|
||||
router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG)
|
||||
assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=_IMAGE_CHAT) == ["vision"]
|
||||
def test_router_picks_the_vision_leg_for_image_requests_without_enable_pre_call_checks():
|
||||
router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG)
|
||||
assert router.enable_pre_call_checks is False
|
||||
for _ in range(5):
|
||||
assert router.get_available_deployment(model="mixed", messages=_IMAGE_CHAT)["model_info"]["id"] == "vision"
|
||||
|
||||
|
||||
def test_pre_call_checks_keep_declared_text_only_legs_for_text_requests():
|
||||
router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG)
|
||||
assert _pre_call_ids(router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=[{"role": "user", "content": "Say hi"}]) == [
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_router_picks_the_vision_leg_for_responses_images_without_enable_pre_call_checks():
|
||||
router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG)
|
||||
for _ in range(5):
|
||||
deployment: Final = await router.async_get_available_deployment(
|
||||
model="mixed", request_kwargs={"input": _IMAGE_RESPONSES_INPUT}
|
||||
)
|
||||
assert deployment["model_info"]["id"] == "vision"
|
||||
|
||||
|
||||
def test_router_sends_an_image_to_the_text_only_leg_when_pre_call_checks_drop_the_vision_leg():
|
||||
router: Final = litellm.Router(
|
||||
model_list=[
|
||||
{"model_name": "mixed", **_VISION_LEG, "model_info": {"id": "vision", "max_input_tokens": 1}},
|
||||
{"model_name": "mixed", **_DECLARED_TEXT_LEG},
|
||||
],
|
||||
routing_strategy="latency-based-routing",
|
||||
enable_pre_call_checks=True,
|
||||
)
|
||||
assert router.get_available_deployment(model="mixed", messages=_IMAGE_CHAT)["model_info"]["id"] == "declared-text"
|
||||
|
||||
|
||||
def test_router_sends_an_image_to_the_text_only_leg_when_the_vision_leg_is_cooling_down():
|
||||
router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG)
|
||||
router.cooldown_cache.add_deployment_to_cooldown(
|
||||
model_id="vision", original_exception=Exception("bad key"), exception_status=401, cooldown_time=60
|
||||
)
|
||||
assert router.get_available_deployment(model="mixed", messages=_IMAGE_CHAT)["model_info"]["id"] == "declared-text"
|
||||
|
||||
|
||||
def test_without_declared_text_only_deployments_passes_a_chosen_deployment_through():
|
||||
assert litellm.Router._without_declared_text_only_deployments(_DECLARED_TEXT_LEG, "mixed", _IMAGE_CHAT, None) is _DECLARED_TEXT_LEG
|
||||
|
||||
|
||||
def test_image_routing_drop_declared_text_only_legs_for_image_requests():
|
||||
router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG)
|
||||
assert _candidate_ids(router, messages=_IMAGE_CHAT) == ["vision"]
|
||||
|
||||
|
||||
def test_image_routing_keep_declared_text_only_legs_for_text_requests():
|
||||
router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG)
|
||||
assert _candidate_ids(router, messages=[{"role": "user", "content": "Say hi"}]) == [
|
||||
"vision",
|
||||
"declared-text",
|
||||
]
|
||||
|
||||
|
||||
def test_pre_call_checks_drop_declared_text_only_legs_for_responses_input_image():
|
||||
router = _pre_call_router(_VISION_LEG, _DECLARED_TEXT_LEG)
|
||||
assert _pre_call_ids(
|
||||
router, (_VISION_LEG, _DECLARED_TEXT_LEG), messages=None, request_kwargs={"input": _IMAGE_RESPONSES_INPUT}
|
||||
) == ["vision"]
|
||||
def test_image_routing_drop_declared_text_only_legs_for_responses_input_image():
|
||||
router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG)
|
||||
assert _candidate_ids(router, request_kwargs={"input": _IMAGE_RESPONSES_INPUT}) == ["vision"]
|
||||
|
||||
|
||||
def test_pre_call_checks_send_image_to_the_group_when_every_leg_is_declared_text_only():
|
||||
def test_image_routing_send_image_to_the_group_when_every_leg_is_declared_text_only():
|
||||
other_text_leg = {**_DECLARED_TEXT_LEG, "model_info": {"id": "declared-text-2", "supports_vision": False}}
|
||||
router = _pre_call_router(_DECLARED_TEXT_LEG, other_text_leg)
|
||||
assert _pre_call_ids(router, (_DECLARED_TEXT_LEG, other_text_leg), messages=_IMAGE_CHAT) == [
|
||||
router: Final = _mixed_router(_DECLARED_TEXT_LEG, other_text_leg)
|
||||
assert _candidate_ids(router, messages=_IMAGE_CHAT) == [
|
||||
"declared-text",
|
||||
"declared-text-2",
|
||||
]
|
||||
|
||||
|
||||
def test_pre_call_checks_keep_undeclared_custom_legs_for_image_requests():
|
||||
def test_image_routing_keep_undeclared_custom_legs_for_image_requests():
|
||||
custom_leg = {
|
||||
"litellm_params": {"model": "hosted_vllm/Qwen2.5-VL-7B-Instruct", "api_base": "http://localhost:8000"},
|
||||
"model_info": {"id": "custom"},
|
||||
}
|
||||
router = _pre_call_router(_VISION_LEG, custom_leg)
|
||||
assert _pre_call_ids(router, (_VISION_LEG, custom_leg), messages=_IMAGE_CHAT) == ["vision", "custom"]
|
||||
router: Final = _mixed_router(_VISION_LEG, custom_leg)
|
||||
assert _candidate_ids(router, messages=_IMAGE_CHAT) == ["vision", "custom"]
|
||||
|
||||
|
||||
def test_pre_call_checks_resolve_vision_through_the_cost_map_and_base_model():
|
||||
def test_image_routing_resolve_vision_through_the_cost_map_and_base_model():
|
||||
catalog_text_leg = {"litellm_params": {"model": "bedrock/deepseek.v3.2"}, "model_info": {"id": "catalog-text"}}
|
||||
azure_text_leg = {
|
||||
"litellm_params": {"model": "azure/my-audio-deployment", "api_base": "https://x.openai.azure.com"},
|
||||
"model_info": {"id": "azure-text", "base_model": "azure/gpt-audio"},
|
||||
}
|
||||
router = _pre_call_router(_VISION_LEG, catalog_text_leg, azure_text_leg)
|
||||
assert _pre_call_ids(router, (_VISION_LEG, catalog_text_leg, azure_text_leg), messages=_IMAGE_CHAT) == ["vision"]
|
||||
router: Final = _mixed_router(_VISION_LEG, catalog_text_leg, azure_text_leg)
|
||||
assert _candidate_ids(router, messages=_IMAGE_CHAT) == ["vision"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue