mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(proxy): skip numbers when walking a request body for s3 media
A body with large token-id lists and any s3:// string or file_id key made the full walk build a node per number, taking seconds per request. The walk and its prefilter now skip int, float, bool and None values.
This commit is contained in:
parent
7aaf7041f6
commit
57e73a8f7d
3 changed files with 35 additions and 5 deletions
|
|
@ -1,7 +1,7 @@
|
|||
import base64
|
||||
import posixpath
|
||||
import re
|
||||
from collections.abc import Callable, Iterator, Mapping, Sequence
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from itertools import accumulate, chain, repeat, takewhile
|
||||
|
|
@ -115,6 +115,10 @@ def _with_embedded_s3_uris(file_id: str) -> Iterator[str]:
|
|||
return chain((file_id,), _s3_uris_embedded_in_file_id(file_id))
|
||||
|
||||
|
||||
def _non_scalar(values: Iterable[object]) -> Iterator[object]:
|
||||
return (value for value in values if type(value) not in _SCALAR_TYPES)
|
||||
|
||||
|
||||
def _sequence_items(value: object) -> Sequence[object] | None:
|
||||
if isinstance(value, tuple):
|
||||
return _OBJECT_TUPLE.validate_python(value)
|
||||
|
|
@ -137,9 +141,15 @@ def _child_nodes(node: _BodyNode) -> Iterator[_BodyNode]:
|
|||
fields: Final = as_str_mapping(node.value)
|
||||
if fields is not None:
|
||||
inside_media_part: Final = node.inside_media_part or _is_media_part(fields, node.key)
|
||||
return (_BodyNode(child, child_key, inside_media_part) for child_key, child in fields.items())
|
||||
return (
|
||||
_BodyNode(child, child_key, inside_media_part)
|
||||
for child_key, child in fields.items()
|
||||
if type(child) not in _SCALAR_TYPES
|
||||
)
|
||||
items: Final = _sequence_items(node.value)
|
||||
return (_BodyNode(item, None, node.inside_media_part) for item in items) if items is not None else iter(())
|
||||
if items is None:
|
||||
return iter(())
|
||||
return (_BodyNode(item, None, node.inside_media_part) for item in _non_scalar(items))
|
||||
|
||||
|
||||
def _next_level(level: tuple[_BodyNode, ...], _: None) -> tuple[_BodyNode, ...]:
|
||||
|
|
@ -201,9 +211,11 @@ def _may_name_an_s3_uri(value: object) -> bool:
|
|||
return is_s3_uri(value)
|
||||
fields: Final = as_str_mapping(value)
|
||||
if fields is not None:
|
||||
return not _KEYS_THAT_MAY_HIDE_AN_S3_URI.isdisjoint(fields) or any(map(_may_name_an_s3_uri, fields.values()))
|
||||
return not _KEYS_THAT_MAY_HIDE_AN_S3_URI.isdisjoint(fields) or any(
|
||||
map(_may_name_an_s3_uri, _non_scalar(fields.values()))
|
||||
)
|
||||
items: Final = _sequence_items(value)
|
||||
return items is not None and not _SCALAR_TYPES.issuperset(map(type, items)) and any(map(_may_name_an_s3_uri, items))
|
||||
return items is not None and any(map(_may_name_an_s3_uri, _non_scalar(items)))
|
||||
|
||||
|
||||
def _may_carry_s3_media(request_body: Mapping[str, object]) -> bool:
|
||||
|
|
|
|||
|
|
@ -191,6 +191,18 @@ def _gemini_turn(*parts: object) -> dict[str, object]:
|
|||
},
|
||||
id="responses-computer-screenshot",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"messages": (
|
||||
{"role": "user", "content": ({"type": "image_url", "image_url": {"url": _MANAGED_UPLOAD}},)},
|
||||
)
|
||||
},
|
||||
id="tuples",
|
||||
),
|
||||
pytest.param(
|
||||
_user_turn(1, 2.5, True, None, {"type": "image_url", "image_url": {"url": _MANAGED_UPLOAD}}),
|
||||
id="numbers-before-the-media-part",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_s3_media_uris_finds_an_s3_object_on_every_media_surface(request_body: dict[str, object]):
|
||||
|
|
|
|||
|
|
@ -232,6 +232,11 @@ def _chat_image(uri: str) -> dict[str, object]:
|
|||
return {"messages": [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": uri}}]}]}
|
||||
|
||||
|
||||
def _chat_image_after_numbers(uri: str) -> dict[str, object]:
|
||||
image_part: Final = {"type": "image_url", "image_url": {"url": uri}}
|
||||
return {"messages": [{"role": "user", "content": [1, 2.5, True, None, image_part]}]}
|
||||
|
||||
|
||||
def _responses_image(uri: str) -> dict[str, object]:
|
||||
return {"input": [{"role": "user", "content": [{"type": "input_image", "image_url": uri}]}]}
|
||||
|
||||
|
|
@ -255,6 +260,7 @@ _S3MediaRoute: TypeAlias = Literal["acompletion", "aresponses", "anthropic_messa
|
|||
_GEMINI_GENERATE_CONTENT_PATH: Final = "/v1beta/models/nova:generateContent"
|
||||
_S3_MEDIA_SURFACES: Final = (
|
||||
pytest.param("acompletion", "/v1/chat/completions", _chat_image, id="chat-completions"),
|
||||
pytest.param("acompletion", "/v1/chat/completions", _chat_image_after_numbers, id="chat-completions-numbers-first"),
|
||||
pytest.param("aresponses", "/v1/responses", _responses_image, id="responses"),
|
||||
pytest.param("anthropic_messages", "/v1/messages", _anthropic_image, id="anthropic-messages"),
|
||||
pytest.param("agenerate_content", _GEMINI_GENERATE_CONTENT_PATH, _gemini_snake_case_image, id="gemini-snake-case"),
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue