scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-08 22:21:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 78

fix(hosted_vllm): forward Omni media URLs instead of downloading them

Browse source 
OpenAI videos never fetches input URLs or invents download size caps. Drop MAX_VIDEO_MEDIA_* and the fetch-and-inline path. Keep MAX_IMAGE_URL_DOWNLOAD_SIZE_MB for chat image handling. Validate Omni reference URLs, then send them as form fields the way OpenAI videos sends input_reference as a file
This commit is contained in:
mubashir1osmani 2026-08-24 17:59:52 -04:00
parent 984b4b5968
commit 999d23dfce
4 changed files with 93 additions and 305 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
litellm/constants.py
View file

@ -69,8 +69,6 @@ DEFAULT_IMAGE_HEIGHT: Final = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300))
# Maps to OpenAI's 50 MB payload limit - requests with images exceeding this size will be rejected
# Set MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0 to disable image URL handling entirely
MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: Final = float(os.getenv("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 50))
MAX_VIDEO_MEDIA_URLS_PER_REQUEST: Final = get_env_int("MAX_VIDEO_MEDIA_URLS_PER_REQUEST", 10)
MAX_VIDEO_MEDIA_URL_TOTAL_DOWNLOAD_SIZE_MB: Final = float(os.getenv("MAX_VIDEO_MEDIA_URL_TOTAL_DOWNLOAD_SIZE_MB", 100))
MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB: Final = int(
os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 1024)
) # 1MB = 1024KB

42
litellm/litellm_core_utils/llm_request_utils.py
View file

@ -14,27 +14,31 @@ def _form_field_value(value: object) -> str:
def _flatten_form_field(key: str, value: object) -> tuple[tuple[str, str], ...]:
work: Final[list[tuple[str, object, int]]] = [(key, value, 0)] # mutable-ok: depth-capped stack, avoids recursion
out: Final[list[tuple[str, str]]] = [] # mutable-ok: local accumulator
while work:
current_key, current_value, depth = work.pop()
pending_fields: Final[ # mutable-ok: depth-capped stack walks nested JSON into multipart names
list[tuple[str, object, int]]
] = [ # mutable-ok: depth-capped stack walks nested JSON into multipart names
(key, value, 0)
]
flat_fields: Final[list[tuple[str, str]]] = [] # mutable-ok: local accumulator
while pending_fields:
current_key, current_value, depth = pending_fields.pop()
if depth > DEFAULT_MAX_RECURSE_DEPTH:
raise ValueError("form field nesting exceeds max depth")
if isinstance(current_value, Mapping):
work.extend(
pending_fields.extend(
(f"{current_key}[{subkey}]", subvalue, depth + 1)
for subkey, subvalue in reversed(tuple(current_value.items()))
)
continue
if isinstance(current_value, (list, tuple)):
work.extend((f"{current_key}[]", entry, depth + 1) for entry in reversed(tuple(current_value)))
pending_fields.extend((f"{current_key}[]", entry, depth + 1) for entry in reversed(tuple(current_value)))
continue
if current_value is None:
continue
serialized = _form_field_value(current_value)
if serialized:
out.append((current_key, serialized))
return tuple(out)
flat_fields.append((current_key, serialized))
return tuple(flat_fields)
def _is_form_scalar(value: object) -> bool:
@ -42,14 +46,18 @@ def _is_form_scalar(value: object) -> bool:
def _flatten_form_data_field(key: str, value: object) -> tuple[tuple[str, str | tuple[str, ...]], ...]:
work: Final[list[tuple[str, object, int]]] = [(key, value, 0)] # mutable-ok: depth-capped stack, avoids recursion
out: Final[list[tuple[str, str | tuple[str, ...]]]] = [] # mutable-ok: local accumulator
while work:
current_key, current_value, depth = work.pop()
pending_fields: Final[ # mutable-ok: depth-capped stack walks nested JSON into multipart names
list[tuple[str, object, int]]
] = [ # mutable-ok: depth-capped stack walks nested JSON into multipart names
(key, value, 0)
]
flat_fields: Final[list[tuple[str, str | tuple[str, ...]]]] = [] # mutable-ok: local accumulator
while pending_fields:
current_key, current_value, depth = pending_fields.pop()
if depth > DEFAULT_MAX_RECURSE_DEPTH:
raise ValueError("form field nesting exceeds max depth")
if isinstance(current_value, Mapping):
work.extend(
pending_fields.extend(
(f"{current_key}[{subkey}]", subvalue, depth + 1)
for subkey, subvalue in reversed(tuple(current_value.items()))
)
@ -58,16 +66,16 @@ def _flatten_form_data_field(key: str, value: object) -> tuple[tuple[str, str |
if all(_is_form_scalar(entry) for entry in current_value):
serialized_fields = tuple(field for entry in current_value if (field := _form_field_value(entry)))
if serialized_fields:
out.append((current_key, serialized_fields))
flat_fields.append((current_key, serialized_fields))
continue
work.extend((f"{current_key}[]", entry, depth + 1) for entry in reversed(tuple(current_value)))
pending_fields.extend((f"{current_key}[]", entry, depth + 1) for entry in reversed(tuple(current_value)))
continue
if current_value is None:
continue
serialized = _form_field_value(current_value)
if serialized:
out.append((current_key, serialized))
return tuple(out)
flat_fields.append((current_key, serialized))
return tuple(flat_fields)
def flatten_form_field_values(*sources: Mapping[str, object] | None) -> tuple[tuple[str, str | tuple[str, ...]], ...]:

198
litellm/llms/hosted_vllm/videos/transformation.py
View file

@ -1,34 +1,21 @@
"""Video generation for Hosted VLLM (vLLM-Omni OpenAI-compatible /v1/videos)."""
import base64
import json
import mimetypes
from collections.abc import Mapping
from dataclasses import dataclass
from io import BufferedReader
from types import MappingProxyType
from typing import Final
from urllib.parse import urlparse
from httpx import Response
from httpx._types import FileTypes, RequestFiles
from litellm.constants import (
MAX_IMAGE_URL_DOWNLOAD_SIZE_MB,
MAX_VIDEO_MEDIA_URL_TOTAL_DOWNLOAD_SIZE_MB,
MAX_VIDEO_MEDIA_URLS_PER_REQUEST,
)
from litellm.images.utils import ImageEditRequestUtils
from litellm.litellm_core_utils.url_utils import SSRFError, safe_get
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
from litellm.llms.openai.videos.transformation import OpenAIVideoConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
from litellm.types.videos.main import VideoCreateOptionalRequestParams
_BYTES_PER_MB: Final = 1024 * 1024
_STREAM_CHUNK_SIZE: Final = 8192
_EXCLUDED_FORM_KEYS: Final = frozenset(
{
"model",
@ -79,28 +66,6 @@ _REFERENCE_URL_KEYS: Final = MappingProxyType(
)
@dataclass(slots=True)
class _MediaDownloadBudget:
remaining_urls: int
remaining_bytes: int
max_bytes_per_url: int
def consume_url_slot(self) -> None:
if self.max_bytes_per_url <= 0:
raise ValueError("remote media URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0)")
if self.remaining_urls < 1:
raise ValueError("too many remote media URL references on one video request")
self.remaining_urls -= 1
def max_read_bytes(self) -> int:
return min(self.max_bytes_per_url, self.remaining_bytes)
def consume_bytes(self, nbytes: int) -> None:
if nbytes > self.remaining_bytes:
raise ValueError("remote media download exceeded the per-request size limit")
self.remaining_bytes -= nbytes
def _serialize_form_value(value: object) -> str:
if isinstance(value, str):
return value
@ -120,91 +85,46 @@ def _maybe_json(value: object) -> object:
return json.loads(stripped)
def _content_type_from_headers(headers: Mapping[str, object]) -> str:
raw: Final = headers.get("content-type", "application/octet-stream")
if not isinstance(raw, str) or not raw.strip():
return "application/octet-stream"
return raw.split(";", 1)[0].strip() or "application/octet-stream"
def _declared_content_length(headers: Mapping[str, object]) -> int | None:
raw: Final = headers.get("content-length")
if not isinstance(raw, str):
return None
stripped: Final = raw.strip()
if not stripped.isdigit():
return None
return int(stripped)
def _read_capped_body(response: Response, max_bytes: int) -> bytes:
declared: Final = _declared_content_length(response.headers)
if declared is not None and declared > max_bytes:
response.close()
raise ValueError("remote media URL Content-Length exceeds the maximum allowed size")
body: Final = bytearray() # mutable-ok: streaming accumulator with a hard cap
for chunk in response.iter_bytes(chunk_size=_STREAM_CHUNK_SIZE):
body.extend(chunk)
if len(body) > max_bytes:
response.close()
raise ValueError("remote media download exceeded the maximum allowed size")
return bytes(body)
def _fetch_url_as_data_url(url: str, client: HTTPHandler, budget: _MediaDownloadBudget) -> str:
budget.consume_url_slot()
response: Final = safe_get(client, url)
response.raise_for_status()
content: Final = _read_capped_body(response, budget.max_read_bytes())
budget.consume_bytes(len(content))
encoded: Final = base64.b64encode(content).decode("ascii")
return f"data:{_content_type_from_headers(response.headers)};base64,{encoded}"
def _inline_reference_item(url_key: str, item: object, client: HTTPHandler, budget: _MediaDownloadBudget) -> object:
if not isinstance(item, Mapping):
return item
url: Final = item.get(url_key)
if not isinstance(url, str):
return item
def _reject_unsafe_media_url(url: str) -> None:
scheme: Final = urlparse(url).scheme.lower()
if scheme in ("", "data"):
return item
return
if scheme not in ("http", "https"):
raise SSRFError(f"URL scheme '{scheme}' is not allowed")
return { # mutable-ok: JSON form field serialized immediately
**item,
url_key: _fetch_url_as_data_url(url, client, budget),
}
validate_url(url)
def _inline_media_reference(
field_name: str, value: object, client: HTTPHandler, budget: _MediaDownloadBudget
) -> object:
def _reject_unsafe_urls_in_item(url_key: str, item: object) -> None:
if not isinstance(item, Mapping):
return
url: Final = item.get(url_key)
if isinstance(url, str):
_reject_unsafe_media_url(url)
def _reject_unsafe_media_urls(field_name: str, value: object) -> None:
url_key: Final = _REFERENCE_URL_KEYS.get(field_name)
if url_key is None:
return value
return
parsed: Final = _maybe_json(value)
if isinstance(parsed, list):
return tuple(_inline_reference_item(url_key, item, client, budget) for item in parsed)
for item in parsed:
_reject_unsafe_urls_in_item(url_key, item)
return
if isinstance(parsed, Mapping):
return _inline_reference_item(url_key, parsed, client, budget)
return value
_reject_unsafe_urls_in_item(url_key, parsed)
def _form_value(key: str, value: object) -> str:
_reject_unsafe_media_urls(key, value)
return _serialize_form_value(value)
def _input_reference_file(reference: object) -> tuple[str, FileTypes]:
content_type: Final = ImageEditRequestUtils.get_image_content_type(reference)
if isinstance(reference, BufferedReader):
reader_name: Final = reference.name
reader_type: Final = mimetypes.guess_type(reader_name)[0] or ImageEditRequestUtils.get_image_content_type(
reference
)
return ("input_reference", (reader_name, reference, reader_type))
fallback_name: Final = getattr(reference, "name", None) or "input_reference.png"
fallback_type: Final = mimetypes.guess_type(str(fallback_name))[0] or ImageEditRequestUtils.get_image_content_type(
reference
)
return ("input_reference", (str(fallback_name), reference, fallback_type))
return ("input_reference", (reference.name, reference, content_type))
return ("input_reference", ("input_reference.png", reference, content_type))
class HostedVLLMVideoConfig(OpenAIVideoConfig):
@ -214,42 +134,6 @@ class HostedVLLMVideoConfig(OpenAIVideoConfig):
https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/videos_api/
"""
def __init__(
self,
media_http_client: HTTPHandler | None = None,
*,
max_media_bytes_per_url: int | None = None,
max_media_bytes_per_request: int | None = None,
max_media_urls_per_request: int | None = None,
) -> None:
super().__init__()
self._media_http_client = media_http_client
self._max_media_bytes_per_url = max_media_bytes_per_url
self._max_media_bytes_per_request = max_media_bytes_per_request
self._max_media_urls_per_request = max_media_urls_per_request
def _media_budget(self) -> _MediaDownloadBudget:
per_url: Final = (
self._max_media_bytes_per_url
if self._max_media_bytes_per_url is not None
else int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * _BYTES_PER_MB)
)
per_request: Final = (
self._max_media_bytes_per_request
if self._max_media_bytes_per_request is not None
else int(MAX_VIDEO_MEDIA_URL_TOTAL_DOWNLOAD_SIZE_MB * _BYTES_PER_MB)
)
url_slots: Final = (
self._max_media_urls_per_request
if self._max_media_urls_per_request is not None
else MAX_VIDEO_MEDIA_URLS_PER_REQUEST
)
return _MediaDownloadBudget(
remaining_urls=url_slots,
remaining_bytes=per_request,
max_bytes_per_url=per_url,
)
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: BaseVideoConfig contract
return [ # mutable-ok: BaseVideoConfig returns list
*super().get_supported_openai_params(model),
@ -307,22 +191,16 @@ class HostedVLLMVideoConfig(OpenAIVideoConfig):
litellm_params: GenericLiteLLMParams,
headers: dict, # mutable-ok: BaseVideoConfig contract
) -> tuple[dict, RequestFiles, str]: # mutable-ok: BaseVideoConfig contract
media_client: Final = self._media_http_client or HTTPHandler(concurrent_limit=1)
media_budget: Final = self._media_budget()
data: Final = { # mutable-ok: BaseVideoConfig contract returns a data dict
"model": model,
"prompt": prompt,
**{ # mutable-ok: spread remaining Omni form fields into that data dict
key: _form_value(key, value)
for key, value in video_create_optional_request_params.items()
if key not in _EXCLUDED_FORM_KEYS and value is not None
},
}
input_reference: Final = video_create_optional_request_params.get("input_reference")
form_files: Final = tuple(
(
key,
(None, _serialize_form_value(_inline_media_reference(key, value, media_client, media_budget))),
)
for key, value in video_create_optional_request_params.items()
if key not in _EXCLUDED_FORM_KEYS and value is not None
)
reference_files: Final = (_input_reference_file(input_reference),) if input_reference is not None else ()
files: Final = (
("model", (None, model)),
("prompt", (None, prompt)),
*form_files,
*reference_files,
)
return {}, files, api_base # mutable-ok: empty data dict; files carry the multipart fields
if input_reference is None:
return data, (), api_base
return data, (_input_reference_file(input_reference),), api_base

156
tests/test_litellm/llms/hosted_vllm/videos/test_hosted_vllm_video_transformation.py
View file

@ -1,6 +1,5 @@
"""Tests for hosted_vllm video generation (vLLM-Omni /v1/videos)."""
import base64
import json
from io import BytesIO
@ -21,10 +20,6 @@ from litellm.types.videos.main import VideoObject
from litellm.utils import ProviderConfigManager
def _form_fields(files: list) -> dict[str, str]:
return {name: value[1] for name, value in files if value[0] is None}
def test_provider_config_registration():
config = ProviderConfigManager.get_provider_video_config(
model="hosted_vllm/MiniMax-H3",
@ -109,27 +104,25 @@ def test_transform_video_create_request_uses_multipart_form_fields():
headers={},
)
assert data == {}
assert url == "http://localhost:8091/v1/videos"
assert files
fields = _form_fields(files)
assert fields["model"] == "MiniMax-H3"
assert fields["prompt"] == "three cats march into a bedroom playing tiny brass instruments"
assert fields["width"] == "1280"
assert fields["height"] == "720"
assert fields["fps"] == "24"
assert fields["num_inference_steps"] == "20"
assert fields["flow_shift"] == "12"
assert fields["seed"] == "1101"
assert fields["aspect_ratio"] == "16:9"
assert json.loads(fields["extra_params"]) == extra_params
assert "extra_headers" not in fields
assert files == ()
assert data["model"] == "MiniMax-H3"
assert data["prompt"] == "three cats march into a bedroom playing tiny brass instruments"
assert data["width"] == "1280"
assert data["height"] == "720"
assert data["fps"] == "24"
assert data["num_inference_steps"] == "20"
assert data["flow_shift"] == "12"
assert data["seed"] == "1101"
assert data["aspect_ratio"] == "16:9"
assert json.loads(data["extra_params"]) == extra_params
assert "extra_headers" not in data
def test_transform_video_create_request_keeps_openai_size_and_seconds():
config = HostedVLLMVideoConfig()
_, files, _ = config.transform_video_create_request(
data, files, _ = config.transform_video_create_request(
model="Wan2.2",
prompt="a mountain lake at sunrise",
api_base="http://localhost:8091/v1/videos",
@ -138,9 +131,9 @@ def test_transform_video_create_request_keeps_openai_size_and_seconds():
headers={},
)
fields = _form_fields(files)
assert fields["seconds"] == "8"
assert fields["size"] == "1280x720"
assert files == ()
assert data["seconds"] == "8"
assert data["size"] == "1280x720"
def test_transform_video_create_request_attaches_input_reference_file():
@ -157,14 +150,12 @@ def test_transform_video_create_request_attaches_input_reference_file():
headers={},
)
assert data == {}
fields = _form_fields(files)
assert fields["width"] == "832"
assert "input_reference" not in fields
assert data["width"] == "832"
assert "input_reference" not in data
reference_parts = [value for name, value in files if name == "input_reference"]
assert len(reference_parts) == 1
filename, content, content_type = reference_parts[0]
assert filename == "input.png"
assert filename == "input_reference.png"
assert content is reference
assert content_type == "image/png"
@ -257,15 +248,9 @@ def test_video_generation_posts_multipart_not_json():
assert request.headers.get("content-type", "").startswith("multipart/form-data")
def test_http_image_reference_is_inlined_as_data_url():
png_bytes = b"fake-png"
def handler(request: httpx.Request) -> httpx.Response:
assert request.url.host == "1.1.1.1"
return httpx.Response(200, content=png_bytes, headers={"content-type": "image/png"})
config = HostedVLLMVideoConfig(media_http_client=_http_handler_for(handler))
_, files, _ = config.transform_video_create_request(
def test_http_image_reference_is_forwarded_not_downloaded():
config = HostedVLLMVideoConfig()
data, files, _ = config.transform_video_create_request(
model="MiniMax-H3",
prompt="a person singing",
api_base="http://localhost:8091/v1/videos",
@ -276,40 +261,15 @@ def test_http_image_reference_is_inlined_as_data_url():
headers={},
)
payload = json.loads(_form_fields(files)["image_reference"])
assert payload["image_url"] == "data:image/png;base64," + base64.b64encode(png_bytes).decode("ascii")
assert files == ()
payload = json.loads(data["image_reference"])
assert payload["image_url"] == "http://1.1.1.1/face.png"
def test_http_audio_reference_json_string_is_inlined_as_data_url():
audio_bytes = b"fake-mp3"
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, content=audio_bytes, headers={"content-type": "audio/mpeg"})
config = HostedVLLMVideoConfig(media_http_client=_http_handler_for(handler))
_, files, _ = config.transform_video_create_request(
model="MiniMax-H3",
prompt="a person singing",
api_base="http://localhost:8091/v1/videos",
video_create_optional_request_params={
"audio_reference": '{"audio_url": "http://1.1.1.1/speech.mp3"}',
},
litellm_params=GenericLiteLLMParams(),
headers={},
)
payload = json.loads(_form_fields(files)["audio_reference"])
assert payload["audio_url"].startswith("data:audio/mpeg;base64,")
assert base64.b64decode(payload["audio_url"].split(",", 1)[1]) == audio_bytes
def test_data_url_image_reference_is_not_fetched():
def handler(request: httpx.Request) -> httpx.Response:
raise AssertionError(f"unexpected fetch of {request.url}")
def test_data_url_image_reference_is_forwarded():
data_url = "data:image/png;base64,AAAA"
config = HostedVLLMVideoConfig(media_http_client=_http_handler_for(handler))
_, files, _ = config.transform_video_create_request(
config = HostedVLLMVideoConfig()
data, files, _ = config.transform_video_create_request(
model="MiniMax-H3",
prompt="a person singing",
api_base="http://localhost:8091/v1/videos",
@ -318,7 +278,8 @@ def test_data_url_image_reference_is_not_fetched():
headers={},
)
assert json.loads(_form_fields(files)["image_reference"])["image_url"] == data_url
assert files == ()
assert json.loads(data["image_reference"])["image_url"] == data_url
def test_metadata_url_in_image_reference_is_rejected():
@ -349,60 +310,3 @@ def test_file_scheme_media_reference_is_rejected():
litellm_params=GenericLiteLLMParams(),
headers={},
)
def _transform_with_references(config: HostedVLLMVideoConfig, **references: object):
return config.transform_video_create_request(
model="MiniMax-H3",
prompt="a person singing",
api_base="http://localhost:8091/v1/videos",
video_create_optional_request_params=references,
litellm_params=GenericLiteLLMParams(),
headers={},
)
def test_oversized_content_length_is_rejected_before_body():
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
content=b"",
headers={"content-type": "video/mp4", "content-length": str(51 * 1024 * 1024)},
)
config = HostedVLLMVideoConfig(media_http_client=_http_handler_for(handler))
with pytest.raises(ValueError, match="Content-Length"):
_transform_with_references(config, video_reference={"video_url": "http://1.1.1.1/clip.mp4"})
def test_streamed_body_over_per_url_cap_is_rejected():
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(
200,
content=iter((b"1234", b"5678")),
headers={"content-type": "image/png"},
)
config = HostedVLLMVideoConfig(
media_http_client=_http_handler_for(handler),
max_media_bytes_per_url=4,
max_media_bytes_per_request=4,
)
with pytest.raises(ValueError, match="exceeded the maximum allowed size"):
_transform_with_references(config, image_reference={"image_url": "http://1.1.1.1/face.png"})
def test_too_many_remote_media_urls_are_rejected():
def handler(request: httpx.Request) -> httpx.Response:
return httpx.Response(200, content=b"ok", headers={"content-type": "image/png"})
config = HostedVLLMVideoConfig(
media_http_client=_http_handler_for(handler),
max_media_urls_per_request=1,
)
with pytest.raises(ValueError, match="too many remote media URL references"):
_transform_with_references(
config,
image_reference={"image_url": "http://1.1.1.1/a.png"},
audio_reference={"audio_url": "http://1.1.1.1/b.mp3"},
)