scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-01 07:03:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 312

fix(router): inspect image content without translating routed requests (#49371)

Browse source 
* fix(router): inspect image content without translating routed requests

* fix(router): preserve screenshot image routing

* test: make monthly budget reset coverage deterministic

---------

Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com>
This commit is contained in:
joshua-berri 2026-10-31 10:44:24 -07:00 • committed by GitHub
parent 34b6256d52
commit 6e27da8af9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 119 additions and 33 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -238,7 +238,7 @@ def is_non_content_values_set(message: AllMessageValues) -> bool:
return any(message.get(key, None) is not None for key in message if key not in ignore_keys)
_IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image"})
_IMAGE_CONTENT_PART_TYPES: Final = ("image_url", "input_image", "image", "computer_screenshot")
_IMAGE_SCAN_MAX_DEPTH: Final = 4
@ -557,15 +557,24 @@ def as_openai_image_part(part: Mapping[str, object]) -> ChatCompletionImageObjec
return {"type": "image_url", "image_url": {"url": url}} if url else None
def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool:
"""Whether any message carries an image content part, across the dialects that reach
pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``,
and Anthropic Messages ``image``, including images nested inside ``tool_result`` blocks."""
return any(
isinstance(content, list) and _content_parts_contain_image(content)
for message in messages
for content in (message.get("content"),)
)
def _message_contains_image_content(message: object) -> bool:
match message:
case {"content": list()}:
return _content_parts_contain_image(content_parts(_CONTENT_PART.validate_python(message)))
case {"type": "function_call_output" | "custom_tool_call_output" | "tool_result", "output": list()}:
output: Final = _CONTENT_PART.validate_python(message)["output"]
return _content_parts_contain_image(_CONTENT_PARTS.validate_python(output))
case {"type": "computer_call_output", "output": dict()}:
return _content_parts_contain_image((_CONTENT_PART.validate_python(message)["output"],))
case _:
return False
def request_contains_image_content(messages: object) -> bool:
"""Inspect native message content without translating or validating the request."""
if not isinstance(messages, (list, tuple)):
return False
return any(_message_contains_image_content(message) for message in _CONTENT_PARTS.validate_python(messages))
def audio_or_image_in_message_content(message: AllMessageValues) -> bool:

9
litellm/router.py
View file

@ -15040,17 +15040,12 @@ class Router(DeprecatedRouterMembers):
@staticmethod
def _request_carries_image(
messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is
messages: object,
request_kwargs: Mapping[str, object],
) -> bool:
from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content
from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages
structured: Final = resolve_structured_messages(
messages=cast(list[dict[str, object]] | None, messages), # cast-ok: chat messages are plain dicts
request_kwargs=cast(dict[str, object], request_kwargs), # cast-ok: the resolver only reads the mapping
)
return bool(structured) and request_contains_image_content(structured)
return request_contains_image_content(messages or request_kwargs.get("messages") or request_kwargs.get("input"))
@staticmethod
def _pre_call_check_requested_chat_params(request_kwargs: Mapping[str, object]) -> frozenset[str]:

6
tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py
View file

@ -2001,6 +2001,12 @@ class TestRequestContainsImageContent:
[{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "x"}}]}],
[{"role": "user", "content": [{"type": "tool_result", "content": [{"type": "text", "text": "ok"}]}]}],
[{"role": "user", "content": None}],
[{"role": "user", "content": [None, "text", {"type": []}]}],
["malformed"],
[None, 5],
"malformed",
5,
None,
[],
],
)

44
tests/unit/proxy/hooks/test_unit_test_max_model_budget_limiter.py
View file

@ -926,19 +926,37 @@ async def test_bedrock_traffic_charges_the_bare_family_name_budget():
@pytest.mark.asyncio
async def test_user_model_budget_window_resets_when_the_period_elapses():
"""
A monthly user budget must start a fresh window once the period elapses,
and the window start must be scoped to that one budget model so a second
model on a shorter period cannot drag it forward.
"""
@pytest.mark.parametrize(
("charged_at", "expired_window_start", "next_reset"),
[
(
datetime(2026, 10, 31, tzinfo=timezone.utc),
datetime(2026, 9, 30, tzinfo=timezone.utc),
datetime(2026, 11, 30, tzinfo=timezone.utc),
),
(
datetime(2024, 3, 1, tzinfo=timezone.utc),
datetime(2024, 1, 31, tzinfo=timezone.utc),
datetime(2024, 4, 1, tzinfo=timezone.utc),
),
(
datetime(2025, 3, 1, tzinfo=timezone.utc),
datetime(2025, 1, 31, tzinfo=timezone.utc),
datetime(2025, 4, 1, tzinfo=timezone.utc),
),
],
ids=["shorter-current-month", "leap-february", "non-leap-february"],
)
async def test_user_model_budget_window_resets_when_the_period_elapses(
charged_at: datetime, expired_window_start: datetime, next_reset: datetime
) -> None:
from litellm.proxy.hooks.model_max_budget_limiter import (
model_budget_spend_cache_key,
model_budget_start_time_cache_key,
)
dual_cache = DualCache()
limiter = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=dual_cache)
limiter = PROXY_VirtualKeyModelMaxBudgetLimiter(dual_cache=dual_cache, clock=charged_at.timestamp)
user_id = "user-1"
user_model_max_budget = {"gpt-4": {"budget_limit": 1.0, "time_period": "1mo"}}
spend_key = model_budget_spend_cache_key(
@ -969,13 +987,10 @@ async def test_user_model_budget_window_resets_when_the_period_elapses():
model="gpt-4",
)
# Age the window past its period. The next charge opens a new window rather
# than adding to the exhausted one.
elapsed = int(timedelta(days=31, minutes=1).total_seconds())
await dual_cache.async_set_cache(
key=start_time_key,
value=datetime.now(timezone.utc).timestamp() - elapsed,
ttl=elapsed,
value=expired_window_start.timestamp(),
ttl=60,
)
await limiter.async_log_success_event(kwargs, response_obj=None, start_time=None, end_time=None)
@ -985,14 +1000,13 @@ async def test_user_model_budget_window_resets_when_the_period_elapses():
entity_id=user_id,
model_max_budget=user_model_max_budget,
cache=dual_cache,
now=charged_at,
) == {
"gpt-4": {
"current_spend": 1.5,
"budget_limit": 1.0,
"time_period": "1mo",
"budget_reset_at": await _reset_at_of_open_window(
dual_cache, Litellm_EntityType.USER, user_id, "gpt-4", "1mo"
),
"budget_reset_at": next_reset.isoformat(),
}
}

64
tests/unit/test_router/test_router.py
View file

@ -31328,7 +31328,7 @@ def _candidate_ids(
request_kwargs: dict[str, object] | None = None,
) -> list[str]:
candidates: Final = router._without_declared_text_only_deployments(
router.get_model_list(model_name="mixed"), "mixed", messages, request_kwargs
router._core_get_model_list(model_name="mixed"), "mixed", messages, request_kwargs
)
return [d["model_info"]["id"] for d in candidates]
@ -31812,3 +31812,65 @@ def test_router_warns_when_a_deployment_enforces_endpoints_without_a_list(caplog
assert "Deployment 'embed-from-config' sets enforce_supported_endpoints without" in caplog.text
assert "Deployment 'embed-from-api' sets enforce_supported_endpoints without" in caplog.text
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["sync", "async"])
@pytest.mark.parametrize("metadata_key", [None, "metadata", "litellm_metadata"])
@pytest.mark.parametrize(
("route", "messages", "request_kwargs"),
[
("/v1/moderations", None, {"input": ["first", "second"]}),
("/v1/embeddings", None, {"messages": []}),
("/v1/messages", ["malformed"], {}),
("/v1/messages", "malformed", {}),
("/v1/messages", 5, {}),
("/v1/messages", None, {"messages": ["malformed"]}),
("/v1/responses", None, {"input": [{"type": "computer_call_output", "output": {"type": "text", "text": "no image"}}]}),
("/v1/responses", None, {"input": [{"type": "computer_call_output", "output": "malformed"}]}),
],
ids=["moderation-list", "embedding-empty-messages", "message-entry", "message-string", "message-number", "kwargs-messages", "computer-output-text", "computer-output-string"],
)
async def test_image_routing_leaves_non_image_and_malformed_requests_to_the_endpoint(
mode: str, metadata_key: str | None, route: str, messages: object, request_kwargs: dict[str, object]
) -> None:
router: Final = _mixed_router(
{**_DECLARED_TEXT_LEG, "litellm_params": {**_DECLARED_TEXT_LEG["litellm_params"], "order": 0}},
{**_VISION_LEG, "litellm_params": {**_VISION_LEG["litellm_params"], "order": 1}},
)
kwargs: Final = {
**request_kwargs,
**({metadata_key: {"user_api_key_request_route": route}} if metadata_key is not None else {}),
}
original: Final = copy.deepcopy(request_kwargs)
deployment: Final = (
await router._core_async_get_available_deployment(model="mixed", messages=messages, request_kwargs=kwargs)
if mode == "async"
else router.get_available_deployment(model="mixed", messages=messages, request_kwargs=kwargs)
)
assert deployment["model_info"]["id"] == "declared-text"
assert all(kwargs[key] == value for key, value in original.items())
@pytest.mark.parametrize(
"request_kwargs",
[
{"messages": _IMAGE_CHAT},
{"input": _IMAGE_RESPONSES_INPUT},
{"messages": [{"role": "user", "content": [{"type": "image", "source": {"type": "url", "url": "https://example.invalid/image.png"}}]}]},
{"messages": [{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "call_image", "content": [_IMAGE_PART]}]}]},
{"input": [{"type": "function_call_output", "call_id": "call_image", "output": [{"type": "input_image", "image_url": "https://example.invalid/image.png"}]}]},
{"input": [{"type": "custom_tool_call_output", "call_id": "call_image", "output": [{"type": "input_image", "image_url": "https://example.invalid/image.png"}]}]},
{"input": [{"type": "computer_call_output", "call_id": "call_image", "output": {"type": "computer_screenshot", "image_url": "https://example.invalid/image.png"}}]},
{"input": [{"role": "user", "content": [{"type": "computer_screenshot", "image_url": "https://example.invalid/image.png"}]}]},
{"input": [{"type": "function_call_output", "call_id": "call_image", "output": [{"type": "computer_screenshot", "image_url": "https://example.invalid/image.png"}]}]},
{"input": [{"type": "custom_tool_call_output", "call_id": "call_image", "output": [{"type": "computer_screenshot", "image_url": "https://example.invalid/image.png"}]}]},
{"input": [{"type": "computer_call_output", "call_id": "call_image", "output": {"type": "computer_screenshot", "url": "https://example.invalid/image.png"}}]},
],
ids=["chat", "responses", "anthropic", "tool-result", "function-output", "custom-output", "computer-output", "screenshot-content", "screenshot-function-output", "screenshot-custom-output", "screenshot-url"],
)
def test_image_routing_inspects_native_request_content_without_translation(request_kwargs: dict[str, object]) -> None:
router: Final = _mixed_router(_VISION_LEG, _DECLARED_TEXT_LEG)
original: Final = copy.deepcopy(request_kwargs)
assert _candidate_ids(router, request_kwargs=request_kwargs) == ["vision"]
assert request_kwargs == original