scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 114

feat(complexity_router): let the LLM classifier see request images (#39825)

Browse source 
The classifier scores extracted text, so a turn whose complexity lives in
its image is invisible to it: a screenshot of a stack trace classifies on
its caption, and an image-only turn flattens to empty text and never
reaches the classifier at all.

classifier_llm_config.vision opts in, off by default, with max_images
bounding what one turn can add. Images are still dropped when the
classifier model is declared supports_vision false. Anthropic and
Responses image parts are rewritten into chat-completions dialect before
they reach the classifier call, since /v1/messages hands the pre-routing
hook its own dialect untranslated.

The local scorer no longer short-circuits heuristic_first or hybrid on a
turn carrying forwarded images, because it reads text alone and its
confidence describes a request it has only partly seen.
This commit is contained in:
tin-berri 2026-09-04 18:59:50 -07:00 • committed by GitHub
parent 2f90a264f6
commit d8ca43a800
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 507 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

39
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -229,6 +229,45 @@ def _content_parts_contain_image(parts: Sequence[object]) -> bool:
return False
def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) -> str | None:
"""Data or remote URL for an Anthropic ``source`` block, in the form chat completions expects."""
source_type: Final = image_source.get("type")
if source_type == "base64":
media_type: Final = image_source.get("media_type") or "image/jpeg"
image_data: Final = image_source.get("data") or ""
return f"data:{media_type};base64,{image_data}" if image_data else None
if source_type == "url":
url: Final = image_source.get("url")
return url if isinstance(url, str) else ""
return None
def _image_part_url(part: Mapping[str, object]) -> str | None:
"""The image URL carried by one content part, whichever of the three dialects wrote it."""
part_type: Final = part.get("type")
if part_type == "image_url":
image_url: Final = part.get("image_url")
if isinstance(image_url, str):
return image_url
return image_url.get("url") if isinstance(image_url, Mapping) else None
if part_type == "input_image":
responses_url: Final = part.get("image_url")
return responses_url if isinstance(responses_url, str) else None
if part_type == "image":
source: Final = part.get("source")
return anthropic_image_source_to_openai_url(source) if isinstance(source, Mapping) else None
return None
def as_openai_image_part(part: Mapping[str, object]) -> ChatCompletionImageObject | None:
"""One image content part rewritten into chat-completions dialect, or None when it is not one.
Rebuilt rather than forwarded so no caller-controlled key beyond the URL rides along.
"""
url: Final = _image_part_url(part)
return {"type": "image_url", "image_url": {"url": url}} if url else None
def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool:
"""Whether any message carries an image content part, across the dialects that reach
pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``,

16
litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py
View file

@ -99,6 +99,7 @@ def create_tool_name_mapping(
from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingChoice
from litellm.litellm_core_utils.prompt_templates.common_utils import (
anthropic_image_source_to_openai_url,
parse_tool_call_arguments,
reasoning_content_from_thinking_blocks,
with_prompt_cache_breakpoint,
@ -1225,20 +1226,7 @@ class LiteLLMAnthropicMessagesAdapter:
"""
if not isinstance(image_source, dict):
return None
source_type: Final = image_source.get("type")
if source_type == "base64":
# Base64 image format
media_type: Final = image_source.get("media_type", "image/jpeg")
image_data: Final = image_source.get("data", "")
if image_data:
return f"data:{media_type};base64,{image_data}"
elif source_type == "url":
# URL-referenced image format
return image_source.get("url", "")
return None
return anthropic_image_source_to_openai_url(image_source)
def _tool_result_content(self, raw_content: object) -> ToolResultContent:
if isinstance(raw_content, str):

144
litellm/router_strategy/complexity_router/complexity_router.py
View file

@ -40,7 +40,10 @@ from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
)
from litellm.litellm_core_utils.internal_call_metadata import forwarded_internal_call_metadata
from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content
from litellm.litellm_core_utils.prompt_templates.common_utils import (
as_openai_image_part,
request_contains_image_content,
)
from litellm.litellm_core_utils.sensitive_data_masker import mask_credentials_in_payload
from litellm.llms.base_llm.base_utils import type_to_response_format_param
from litellm.router_strategy.adaptive_router.classifier import classify_prompt
@ -48,7 +51,11 @@ from litellm.router_strategy.complexity_router.tier_predictor import (
TierSuccessPredictor,
resolve_tier_artifact,
)
from litellm.types.llms.openai import AllMessageValues
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionImageObject,
ChatCompletionTextObject,
)
from litellm.types.utils import (
AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
ModelResponse,
@ -435,6 +442,23 @@ def _strip_reminder_blocks(text: str, marker_pairs: tuple[tuple[str, str], ...]
return " ".join(kept for a, b in zip(keep_from, keep_to) if (kept := text[a:b].strip()))
def _inline_image_part(part: Mapping[str, object]) -> ChatCompletionImageObject | None:
"""One image content part safe to hand the classifier, or None.
Inline data URIs only. A remote URL is caller-controlled and provider adapters do not uniformly
delegate fetching to the provider: gigachat's file handler downloads any non-data URL with
`client.get` from the proxy host, so forwarding one would let a key scoped to this router aim a
proxy-side request at an internal address, on a call the caller never asked for. The routed
model still receives the original URL exactly as before.
"""
converted: Final = as_openai_image_part(part)
if converted is None:
return None
image_url: Final = converted["image_url"]
url: Final = image_url if isinstance(image_url, str) else image_url.get("url", "")
return converted if url.startswith("data:") else None
def _human_text(content: object, marker_pairs: tuple[tuple[str, str], ...] = _DEFAULT_REMINDER_MARKERS) -> str:
"""Message content as the text a human wrote, with complete reminder blocks removed.
@ -1592,6 +1616,10 @@ class ComplexityRouter(CustomLogger):
threshold check alone would hand that traffic to the cheapest model without ever consulting
the classifier. Scores also go negative when simple indicators fire, so a score threshold
would reject exactly the trivial prompts this path exists to serve.
A turn carrying images the classifier would see is never decided cheaply: the scorer reads
text alone, so its confidence describes a request it has only partly seen, and a trivial
caption beside a screenshot is exactly the misrouting vision classification exists to stop.
"""
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
@ -1599,6 +1627,7 @@ class ComplexityRouter(CustomLogger):
decided_cheaply: Final = (
threshold is not None
and bool(signals)
and not self._classifier_image_parts(messages)
and self._active_tier_severity(tier) <= self._active_tier_severity(threshold)
)
if decided_cheaply:
@ -1623,11 +1652,43 @@ class ComplexityRouter(CustomLogger):
tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
margin: Final = self.config.hybrid_boundary_margin
decided: Final = margin is not None and bool(signals) and not self._is_near_tier_boundary(score, margin)
decided: Final = (
margin is not None
and bool(signals)
and not self._classifier_image_parts(messages)
and not self._is_near_tier_boundary(score, margin)
)
if decided:
return ClassificationOutcome(tier=tier, score=score, signals=signals, cause="hybrid_short_circuit")
return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages, scored=scored)
def _classifier_image_parts(
self, messages: Sequence[Mapping[str, object]] | None
) -> tuple[ChatCompletionImageObject, ...]:
"""Images from the newest user turn to hand the classifier, capped by max_images.
Empty unless the operator opted in AND the classifier model is declared vision-capable, so
every other deployment keeps today's text-only payload byte for byte. Only the newest user
turn is read: earlier turns are context the classifier already gets as quoted text, and an
image nested in a tool_result is tool output rather than the ask being classified.
Remote-URL images are left out entirely; `_inline_image_part` carries why.
"""
llm_config: Final = self.config.classifier_llm_config
if llm_config is None or not llm_config.vision.enabled or not self.config.uses_llm_classifier or not messages:
return ()
if not self._model_declares_vision_support(llm_config.model):
return ()
newest_user_turn: Final = next((msg for msg in reversed(messages) if msg.get("role") == "user"), None)
content: Final = newest_user_turn.get("content") if newest_user_turn is not None else None
if not isinstance(content, list):
return ()
return tuple(
islice(
(part for raw in content if isinstance(raw, Mapping) and (part := _inline_image_part(raw)) is not None),
llm_config.vision.max_images,
)
)
async def _llm_classifier_outcome(
self,
prompt: str,
@ -1865,9 +1926,18 @@ class ComplexityRouter(CustomLogger):
}
turn_off_message_logging: Final = _effective_turn_off_message_logging(request_kwargs)
image_parts: Final = self._classifier_image_parts(messages)
user_content: Final[str | Sequence[ChatCompletionTextObject | ChatCompletionImageObject]] = (
[ # mutable-ok: SDK request payload content list is built once
{"type": "text", "text": user_payload},
*image_parts,
]
if image_parts
else user_payload
)
messages_for_call: Final[list[AllMessageValues]] = [ # mutable-ok: SDK request payload list is built once
{"role": "system", "content": classifier_system_prompt},
{"role": "user", "content": user_payload},
{"role": "user", "content": user_content},
]
response_format: Final = classifier_response_format
classifier_call_params: Mapping[str, str] = EMPTY_MAPPING
@ -2558,31 +2628,53 @@ class ComplexityRouter(CustomLogger):
return pinned_model
return self.get_model_for_tier(escalated_tier)
def _model_accepts_image_input(self, model_name: str) -> bool:
"""Whether a routed model or pool entry can serve an image request.
def _vision_verdicts(self, model_name: str) -> tuple[bool | None, ...]:
"""Declared vision support per deployment serving the name: True, False, or None when
nothing declares either way.
Resolved through the deployments that would actually serve the name; a name with no
deployment on the router is served by the SDK directly and is checked against the model
cost map itself. Only an explicit supports_vision false excludes, a deployment-level
model_info override first and the map otherwise, so unmapped custom names stay routable.
cost map itself. A deployment-level model_info override wins over the map.
One verdict set, two readings, because the two callers fail in opposite directions.
Routing a user's image asks whether anything RULES IT OUT, so an undeclared model stays
eligible and unmapped custom names keep routing. Handing an image to the classifier asks
whether something RULES IT IN: an undeclared model that turns out to be text-only rejects
every image request, and that rejection is swallowed by the classifier's own fallback, so
the router quietly serves all image traffic from the fallback tier and pays for the failed
call each time. An undeclared model instead keeps today's text-only payload, which is a
visible no-op the operator fixes by declaring supports_vision on the deployment.
"""
from litellm.utils import is_vision_explicitly_disabled, supports_vision
def model_verdict(model: str) -> bool | None:
if supports_vision(model):
return True
return False if is_vision_explicitly_disabled(model) else None
def deployment_verdict(deployment: Mapping[str, Any]) -> bool | None:
declared: Final = (deployment.get("model_info") or EMPTY_MAPPING).get("supports_vision")
if declared is not None:
return declared is True
return model_verdict((deployment.get("litellm_params") or EMPTY_MAPPING).get("model") or model_name)
deployments: Final = self.litellm_router_instance.get_model_list(model_name=model_name)
if not deployments:
return (model_verdict(model_name),)
return tuple(deployment_verdict(deployment) for deployment in deployments)
def _model_accepts_image_input(self, model_name: str) -> bool:
"""Whether a routed model or pool entry can serve an image request.
A multi-deployment group must accept on EVERY deployment: the router picks a deployment
inside the group after this gate runs, so a mixed group marked eligible could still hand
the image to its text-only member and fail with the exact 400 the gate exists to prevent.
"""
from litellm.utils import is_vision_explicitly_disabled
return all(verdict is not False for verdict in self._vision_verdicts(model_name))
def deployment_accepts(deployment: Mapping[str, Any]) -> bool:
declared: Final = (deployment.get("model_info") or EMPTY_MAPPING).get("supports_vision")
if declared is not None:
return declared is True
litellm_model: Final = (deployment.get("litellm_params") or EMPTY_MAPPING).get("model") or model_name
return not is_vision_explicitly_disabled(litellm_model)
deployments: Final = self.litellm_router_instance.get_model_list(model_name=model_name)
if not deployments:
return not is_vision_explicitly_disabled(model_name)
return all(deployment_accepts(deployment) for deployment in deployments)
def _model_declares_vision_support(self, model_name: str) -> bool:
"""Whether every deployment serving the name is declared vision-capable."""
return all(verdict is True for verdict in self._vision_verdicts(model_name))
def _modality_eligible_models(self) -> frozenset[str]:
"""Every configured pool entry, plus default_model, that can serve an image request."""
@ -3374,8 +3466,9 @@ class ComplexityRouter(CustomLogger):
has_original_messages: Final = messages is not None and len(messages) > 0
user_message, system_prompt = _extract_current_ask_and_system_prompt(resolved_messages, self._reminder_markers)
classifier_images: Final = self._classifier_image_parts(resolved_messages)
if user_message is None:
if user_message is None and not classifier_images:
verbose_router_logger.debug("ComplexityRouter: No user message found, routing to default model")
default_model_first: Final = not self.config.plugins and self.config.default_model
if default_model_first:
@ -3402,6 +3495,7 @@ class ComplexityRouter(CustomLogger):
),
)
ask: Final = user_message or ""
newest_ask: Final = _newest_turn_ask(resolved_messages, self._reminder_markers)
escalation_keyword: Final = self._matched_escalation_keyword(newest_ask) if newest_ask is not None else None
# Resolved here rather than beside the classifier because the keyword-override path below
@ -3439,7 +3533,7 @@ class ComplexityRouter(CustomLogger):
),
)
override: Final = await self._resolve_keyword_tier_override(user_message, request_kwargs)
override: Final = await self._resolve_keyword_tier_override(ask, request_kwargs)
if override is not None:
keyword_bumped_tier: Final = (
self._escalate_tier(override.tier) if escalation_keyword is not None else override.tier
@ -3486,9 +3580,7 @@ class ComplexityRouter(CustomLogger):
outcome: Final = (
ClassificationOutcome(tier=housekeeping_tier, score=None, signals=("housekeeping",), cause="housekeeping")
if housekeeping_tier is not None
else await self.aclassify(
user_message, system_prompt, request_kwargs, resolved_messages, raw_messages=messages
)
else await self.aclassify(ask, system_prompt, request_kwargs, resolved_messages, raw_messages=messages)
)
tier, score, signals = outcome.tier, outcome.score, outcome.signals
classified_tier: Final = tier
@ -3558,7 +3650,7 @@ class ComplexityRouter(CustomLogger):
# under is not a floor.
routed_model = self._soft_floor_pick(
tier,
user_message,
ask,
request_kwargs,
hard_floor=tier if context_original_tier is not None else plan_floor,
hard_ceiling=housekeeping_ceiling,

35
litellm/router_strategy/complexity_router/config.py
View file

@ -442,12 +442,47 @@ DEFAULT_TIER_MODELS: Final[dict[str, str]] = {
}
class ClassifierVisionConfig(BaseModel):
"""Whether the LLM classifier sees the images on the request it is classifying.
Off by default because images cost far more than the text ask they arrive with, and the
classifier runs on every request. A turn whose complexity lives in the image ("what is wrong in
this stack trace screenshot") is invisible to a text-only classifier, which is what this buys.
"""
enabled: bool = Field(
default=False,
description=(
"Forward image content to the classifier. Requires a classifier model declared "
"supports_vision, on the deployment's model_info or in the model cost map; images stay "
"stripped otherwise, so a classifier that cannot read them is never sent one. Declare "
"model_info.supports_vision on the deployment to enable a model the cost map does not "
"describe. Only inline data: URIs are forwarded. A request whose images are http(s) "
"URLs still classifies on its text alone, because some providers fetch such a URL from "
"the proxy rather than the provider, which would let a caller aim a proxy-side request "
"at an address of their choosing."
),
)
max_images: int = Field(
default=1,
ge=1,
description=(
"How many images from the newest user turn to forward, in wire order. Bounds the added "
"cost of a turn that attaches many images. Images on earlier turns are never forwarded."
),
)
class ClassifierLLMConfig(BaseModel):
"""Configuration for the LLM-based complexity classifier."""
model: str = Field(
description="Model name (from the router's model_list) to call for classification",
)
vision: ClassifierVisionConfig = Field(
default_factory=ClassifierVisionConfig,
description="Whether the classifier sees images on the request, and how many",
)
reasoning_effort: REASONING_EFFORT | None = Field(
default=None,
description=(

274
tests/test_litellm/router_strategy/test_complexity_router.py
View file

@ -12429,3 +12429,277 @@ class TestTierHealthFailover:
for _ in range(20)
]
assert {r.model for r in results} == {"live-c"}
ANTHROPIC_IMG_PART = {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "aGk="}}
RESPONSES_IMG_PART = {"type": "input_image", "image_url": "data:image/png;base64,aGk="}
class TestClassifierVision:
"""classifier_llm_config.vision: what the LLM classifier is shown for an image-bearing turn."""
TIERS = {"SIMPLE": "t-simple", "MEDIUM": "t-medium", "COMPLEX": "t-complex", "REASONING": "t-reasoning"}
@staticmethod
def _router(mock_router_instance, *, vision, classifier_declares_vision=True, classifier_type="llm", **extra):
def get_model_list(model_name=None):
if model_name != "clf":
return [{"model_name": model_name, "litellm_params": {"model": "openai/gpt-4o"}}]
declared = classifier_declares_vision
return [
{
"model_name": "clf",
"litellm_params": {"model": "openai/unmapped-classifier"},
"model_info": {} if declared is None else {"supports_vision": declared},
}
]
mock_router_instance.get_model_list = get_model_list
classifier_llm_config = {"model": "clf", "circuit_breaker_enabled": False}
return ComplexityRouter(
model_name="vision-classifier-router",
litellm_router_instance=mock_router_instance,
complexity_router_config={
"classifier_type": classifier_type,
"classifier_llm_config": (
classifier_llm_config if vision is None else {**classifier_llm_config, "vision": vision}
),
"tiers": dict(TestClassifierVision.TIERS),
**extra,
},
)
@staticmethod
def _classifier_user_content(mock_router_instance):
return mock_router_instance.acompletion.call_args.kwargs["messages"][-1]["content"]
@staticmethod
def _turn(*parts):
return [{"role": "user", "content": list(parts)}]
@pytest.fixture(autouse=True)
def _classifier_answers_complex(self, mock_router_instance):
mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "COMPLEX"}'))
@pytest.mark.asyncio
@pytest.mark.parametrize(
"vision, classifier_declares_vision",
[
(None, True),
({"enabled": False}, True),
({"enabled": True}, False),
({"enabled": True}, None),
],
ids=["vision_unset", "vision_disabled", "classifier_declared_text_only", "classifier_undeclared"],
)
async def test_payload_stays_text_only(self, mock_router_instance, vision, classifier_declares_vision):
"""Off, or a classifier not declared vision-capable, keeps the plain-string payload.
The undeclared case is the polarity. A text-only classifier handed an image rejects the
call, the rejection is swallowed by the classifier's own fallback, and every image request
then serves from the fallback tier while still paying for the failed call. Staying text-only
is instead a visible no-op the operator fixes by declaring supports_vision.
"""
router = self._router(
mock_router_instance, vision=vision, classifier_declares_vision=classifier_declares_vision
)
await router.async_pre_routing_hook(
model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART)
)
content = self._classifier_user_content(mock_router_instance)
assert isinstance(content, str)
assert "what is this" in content
@pytest.mark.asyncio
async def test_deployment_model_info_enables_a_classifier_the_cost_map_does_not_describe(
self, mock_router_instance
):
"""The escape hatch for an unmapped classifier name, and the reason undeclared can stay off.
`_router` gives every deployment an `openai/unmapped-*` litellm_params model, so nothing in
the cost map declares it and the verdict comes only from model_info.
"""
router = self._router(mock_router_instance, vision={"enabled": True}, classifier_declares_vision=True)
await router.async_pre_routing_hook(
model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART)
)
assert [b["type"] for b in self._classifier_user_content(mock_router_instance)] == ["text", "image_url"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"part",
[IMG_PART, ANTHROPIC_IMG_PART, RESPONSES_IMG_PART],
ids=["chat_completions", "anthropic_messages", "responses"],
)
async def test_image_reaches_the_classifier_in_chat_completions_dialect(self, mock_router_instance, part):
"""Every surface's dialect arrives as a chat-completions image_url on the classifier call.
/v1/messages hands the hook an Anthropic image block untranslated, so forwarding verbatim
would send the classifier a content part its own request dialect has no meaning for.
"""
router = self._router(mock_router_instance, vision={"enabled": True})
await router.async_pre_routing_hook(
model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, part)
)
content = self._classifier_user_content(mock_router_instance)
assert [block["type"] for block in content] == ["text", "image_url"]
assert content[1]["image_url"] == {"url": "data:image/png;base64,aGk="}
assert "what is this" in content[0]["text"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"part",
[
{"type": "image_url", "image_url": {"url": "http://169.254.169.254/latest/meta-data/"}},
{"type": "image_url", "image_url": {"url": "https://example.internal/secret.png"}},
{"type": "input_image", "image_url": "https://example.internal/secret.png"},
{"type": "image", "source": {"type": "url", "url": "https://example.internal/secret.png"}},
],
ids=["metadata_service", "chat_completions", "responses", "anthropic"],
)
async def test_remote_url_images_are_never_forwarded(self, mock_router_instance, part):
"""A caller-supplied URL must not reach an internal call the caller did not ask for.
Provider adapters do not uniformly delegate fetching: gigachat downloads any non-data URL
from the proxy host, so forwarding one would turn a router-scoped key into a proxy-side GET
at an address of the caller's choosing.
"""
router = self._router(mock_router_instance, vision={"enabled": True})
await router.async_pre_routing_hook(
model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, part)
)
assert isinstance(self._classifier_user_content(mock_router_instance), str)
@pytest.mark.asyncio
async def test_remote_url_image_only_turn_does_not_reach_the_classifier(self, mock_router_instance):
"""With nothing forwardable left, the turn stays unclassifiable rather than sending the URL."""
router = self._router(mock_router_instance, vision={"enabled": True})
response = await router.async_pre_routing_hook(
model="m",
request_kwargs={},
messages=self._turn({"type": "image_url", "image_url": {"url": "https://example.internal/x.png"}}),
)
assert response.routing_decision["cause"] == "default_fallback"
mock_router_instance.acompletion.assert_not_awaited()
@pytest.mark.asyncio
async def test_image_only_turn_is_classified_instead_of_falling_back(self, mock_router_instance):
"""A turn carrying only an image reaches the classifier rather than the default model.
It flattens to empty text, so before this it never reached the classifier at all and was
routed as default_fallback on text the request never contained.
"""
router = self._router(mock_router_instance, vision={"enabled": True})
response = await router.async_pre_routing_hook(
model="m", request_kwargs={}, messages=self._turn(IMG_PART)
)
assert response.routing_decision["cause"] == "llm_classifier"
assert response.model == "t-complex"
assert [block["type"] for block in self._classifier_user_content(mock_router_instance)] == [
"text",
"image_url",
]
@pytest.mark.asyncio
async def test_image_only_turn_still_falls_back_when_vision_is_off(self, mock_router_instance):
router = self._router(mock_router_instance, vision={"enabled": False})
response = await router.async_pre_routing_hook(
model="m", request_kwargs={}, messages=self._turn(IMG_PART)
)
assert response.routing_decision["cause"] == "default_fallback"
mock_router_instance.acompletion.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("max_images, expected", [(1, 1), (2, 2), (5, 3)])
async def test_max_images_caps_what_is_forwarded(self, mock_router_instance, max_images, expected):
router = self._router(mock_router_instance, vision={"enabled": True, "max_images": max_images})
images = [dict(IMG_PART, image_url={"url": f"data:image/png;base64,{n}"}) for n in ("a", "b", "c")]
await router.async_pre_routing_hook(
model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "look"}, *images)
)
content = self._classifier_user_content(mock_router_instance)
forwarded = [block for block in content if block["type"] == "image_url"]
assert len(forwarded) == expected
assert [block["image_url"]["url"] for block in forwarded] == [
f"data:image/png;base64,{n}" for n in ("a", "b", "c")[:expected]
]
@pytest.mark.asyncio
async def test_earlier_turn_images_are_not_forwarded(self, mock_router_instance):
"""Only the newest user turn's images ride along, so history cannot inflate every call.
The two turns carry different images on purpose: identical ones would pass this assertion
whichever turn the helper read.
"""
older = dict(IMG_PART, image_url={"url": "data:image/png;base64,OLDER"})
newer = dict(IMG_PART, image_url={"url": "data:image/png;base64,NEWER"})
router = self._router(mock_router_instance, vision={"enabled": True, "max_images": 5})
await router.async_pre_routing_hook(
model="m",
request_kwargs={},
messages=[
{"role": "user", "content": [{"type": "text", "text": "first"}, older]},
{"role": "assistant", "content": "ok"},
{"role": "user", "content": [{"type": "text", "text": "second"}, newer]},
],
)
content = self._classifier_user_content(mock_router_instance)
forwarded = [block for block in content if block["type"] == "image_url"]
assert [block["image_url"]["url"] for block in forwarded] == ["data:image/png;base64,NEWER"]
@pytest.mark.asyncio
async def test_logged_request_body_matches_what_was_sent(self, mock_router_instance):
"""proxy_server_request is the logged copy of the classifier call and must not drift."""
router = self._router(mock_router_instance, vision={"enabled": True})
await router.async_pre_routing_hook(
model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART)
)
call_kwargs = mock_router_instance.acompletion.call_args.kwargs
assert call_kwargs["proxy_server_request"]["body"]["messages"] == call_kwargs["messages"]
SHORT_CIRCUIT_ARMS = [
("heuristic_first", {"heuristic_first_max_tier": "SIMPLE"}, "heuristic_first_short_circuit"),
("hybrid", {"hybrid_boundary_margin": 0.05}, "hybrid_short_circuit"),
]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"classifier_type, extra, short_circuit_cause", SHORT_CIRCUIT_ARMS, ids=["heuristic_first", "hybrid"]
)
async def test_local_scorer_cannot_short_circuit_a_turn_it_cannot_see(
self, mock_router_instance, classifier_type, extra, short_circuit_cause
):
"""The scorer reads text alone, so its confidence is not a verdict on an image turn.
Both arms are tuned so the scorer WOULD short-circuit on this exact text, which is what
makes the image the only variable; a margin loose enough to leave the score undecided
would pass whether or not the guard exists.
"""
router = self._router(
mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra
)
response = await router.async_pre_routing_hook(
model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART)
)
assert response.routing_decision["cause"] == "llm_classifier"
@pytest.mark.asyncio
@pytest.mark.parametrize(
"classifier_type, extra, short_circuit_cause", SHORT_CIRCUIT_ARMS, ids=["heuristic_first", "hybrid"]
)
async def test_local_scorer_still_short_circuits_without_images(
self, mock_router_instance, classifier_type, extra, short_circuit_cause
):
"""The negative class: same router, same text, no image, and the scorer still decides."""
router = self._router(
mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra
)
response = await router.async_pre_routing_hook(
model="m", request_kwargs={}, messages=[{"role": "user", "content": "what is this"}]
)
assert response.routing_decision["cause"] == short_circuit_cause
mock_router_instance.acompletion.assert_not_awaited()
def test_max_images_must_be_positive(self):
with pytest.raises(ValidationError):
ClassifierLLMConfig(model="clf", vision={"enabled": True, "max_images": 0})

6
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.test.ts
View file

@ -592,6 +592,12 @@ describe("classifier prompt and fallback", () => {
timeout_ms: 1,
});
});
it.each([{}, { system_prompt: "x" }])("normalizeClassifierLlmConfig carries vision through %o", (extra) => {
const base = { model: "m", timeout_ms: 1, ...extra };
const vision = { enabled: true, max_images: 2 };
expect(normalizeClassifierLlmConfig({ ...base, vision })).toEqual({ ...base, vision });
});
});
describe("tier labels", () => {

11
ui/litellm-dashboard/src/components/add_model/build_complexity_router_config.ts
View file

@ -1,4 +1,7 @@
import { KeywordTierRule } from "./KeywordTierRules";
type ClassifierLLMConfigWire = ClassifierLLMConfig & { vision?: { enabled?: boolean; max_images?: number } };
import type { ModelGroup } from "../llm_calls/fetch_models";
import {
type CustomTierSet,
@ -61,7 +64,8 @@ export const normalizeClassifierLlmConfig = ({
reasoning_effort,
classification_rubric,
system_prompt,
}: ClassifierLLMConfig): ClassifierLLMConfig =>
vision,
}: ClassifierLLMConfigWire): ClassifierLLMConfigWire =>
system_prompt?.trim()
? {
model,
@ -69,6 +73,7 @@ export const normalizeClassifierLlmConfig = ({
...(circuit_breaker_enabled !== undefined && { circuit_breaker_enabled }),
...(circuit_breaker_cooldown_seconds !== undefined && { circuit_breaker_cooldown_seconds }),
...(reasoning_effort && { reasoning_effort }),
...(vision && { vision }),
system_prompt,
}
: {
@ -78,6 +83,7 @@ export const normalizeClassifierLlmConfig = ({
...(circuit_breaker_cooldown_seconds !== undefined && { circuit_breaker_cooldown_seconds }),
...(reasoning_effort && { reasoning_effort }),
...(classification_rubric && { classification_rubric }),
...(vision && { vision }),
};
interface ScorerKnobInputs {
@ -324,7 +330,7 @@ export const getSemanticConfigError = ({
};
interface CustomTierWireFieldInputs {
classifierLlmConfig: ClassifierLLMConfig | undefined;
classifierLlmConfig: ClassifierLLMConfigWire | undefined;
planModeMinTierId: string | undefined;
classificationPrompt: string | undefined;
classificationExamples: string | undefined;
@ -356,6 +362,7 @@ export const customTierWireFields = (
circuit_breaker_cooldown_seconds: classifierLlmConfig.circuit_breaker_cooldown_seconds,
}),
...(classifierLlmConfig.reasoning_effort && { reasoning_effort: classifierLlmConfig.reasoning_effort }),
...(classifierLlmConfig.vision && { vision: classifierLlmConfig.vision }),
},
}),
session_affinity: false,

24
ui/litellm-dashboard/src/lib/http/schema.d.ts generated vendored
View file

@ -25305,6 +25305,30 @@ export interface components {
* @default 3000
*/
timeout_ms: number;
/** @description Whether the classifier sees images on the request, and how many */
vision?: components["schemas"]["ClassifierVisionConfig"];
};
/**
* ClassifierVisionConfig
* @description Whether the LLM classifier sees the images on the request it is classifying.
*
* Off by default because images cost far more than the text ask they arrive with, and the
* classifier runs on every request. A turn whose complexity lives in the image ("what is wrong in
* this stack trace screenshot") is invisible to a text-only classifier, which is what this buys.
*/
ClassifierVisionConfig: {
/**
* Enabled
* @description Forward image content to the classifier. Requires a classifier model declared supports_vision, on the deployment's model_info or in the model cost map; images stay stripped otherwise, so a classifier that cannot read them is never sent one. Declare model_info.supports_vision on the deployment to enable a model the cost map does not describe. Only inline data: URIs are forwarded. A request whose images are http(s) URLs still classifies on its text alone, because some providers fetch such a URL from the proxy rather than the provider, which would let a caller aim a proxy-side request at an address of their choosing.
* @default false
*/
enabled: boolean;
/**
* Max Images
* @description How many images from the newest user turn to forward, in wire order. Bounds the added cost of a turn that attaches many images. Images on earlier turns are never forwarded.
* @default 1
*/
max_images: number;
};
/**
* CloudZeroExportRequest