From 999d23dfce262a7da202ff1341f011156bfa5317 Mon Sep 17 00:00:00 2001 From: mubashir1osmani Date: Mon, 24 Aug 2026 17:59:52 -0400 Subject: [PATCH] fix(hosted_vllm): forward Omni media URLs instead of downloading them OpenAI videos never fetches input URLs or invents download size caps. Drop MAX_VIDEO_MEDIA_* and the fetch-and-inline path. Keep MAX_IMAGE_URL_DOWNLOAD_SIZE_MB for chat image handling. Validate Omni reference URLs, then send them as form fields the way OpenAI videos sends input_reference as a file --- litellm/constants.py | 2 - .../litellm_core_utils/llm_request_utils.py | 42 ++-- .../llms/hosted_vllm/videos/transformation.py | 198 ++++-------------- .../test_hosted_vllm_video_transformation.py | 156 +++----------- 4 files changed, 93 insertions(+), 305 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index c8ece1bc8f4..d6a00294004 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -69,8 +69,6 @@ DEFAULT_IMAGE_HEIGHT: Final = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300)) # Maps to OpenAI's 50 MB payload limit - requests with images exceeding this size will be rejected # Set MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0 to disable image URL handling entirely MAX_IMAGE_URL_DOWNLOAD_SIZE_MB: Final = float(os.getenv("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 50)) -MAX_VIDEO_MEDIA_URLS_PER_REQUEST: Final = get_env_int("MAX_VIDEO_MEDIA_URLS_PER_REQUEST", 10) -MAX_VIDEO_MEDIA_URL_TOTAL_DOWNLOAD_SIZE_MB: Final = float(os.getenv("MAX_VIDEO_MEDIA_URL_TOTAL_DOWNLOAD_SIZE_MB", 100)) MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB: Final = int( os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 1024) ) # 1MB = 1024KB diff --git a/litellm/litellm_core_utils/llm_request_utils.py b/litellm/litellm_core_utils/llm_request_utils.py index c6bf798d179..04824a5bf39 100644 --- a/litellm/litellm_core_utils/llm_request_utils.py +++ b/litellm/litellm_core_utils/llm_request_utils.py @@ -14,27 +14,31 @@ def _form_field_value(value: object) -> str: def _flatten_form_field(key: str, value: object) -> tuple[tuple[str, str], ...]: - work: Final[list[tuple[str, object, int]]] = [(key, value, 0)] # mutable-ok: depth-capped stack, avoids recursion - out: Final[list[tuple[str, str]]] = [] # mutable-ok: local accumulator - while work: - current_key, current_value, depth = work.pop() + pending_fields: Final[ # mutable-ok: depth-capped stack walks nested JSON into multipart names + list[tuple[str, object, int]] + ] = [ # mutable-ok: depth-capped stack walks nested JSON into multipart names + (key, value, 0) + ] + flat_fields: Final[list[tuple[str, str]]] = [] # mutable-ok: local accumulator + while pending_fields: + current_key, current_value, depth = pending_fields.pop() if depth > DEFAULT_MAX_RECURSE_DEPTH: raise ValueError("form field nesting exceeds max depth") if isinstance(current_value, Mapping): - work.extend( + pending_fields.extend( (f"{current_key}[{subkey}]", subvalue, depth + 1) for subkey, subvalue in reversed(tuple(current_value.items())) ) continue if isinstance(current_value, (list, tuple)): - work.extend((f"{current_key}[]", entry, depth + 1) for entry in reversed(tuple(current_value))) + pending_fields.extend((f"{current_key}[]", entry, depth + 1) for entry in reversed(tuple(current_value))) continue if current_value is None: continue serialized = _form_field_value(current_value) if serialized: - out.append((current_key, serialized)) - return tuple(out) + flat_fields.append((current_key, serialized)) + return tuple(flat_fields) def _is_form_scalar(value: object) -> bool: @@ -42,14 +46,18 @@ def _is_form_scalar(value: object) -> bool: def _flatten_form_data_field(key: str, value: object) -> tuple[tuple[str, str | tuple[str, ...]], ...]: - work: Final[list[tuple[str, object, int]]] = [(key, value, 0)] # mutable-ok: depth-capped stack, avoids recursion - out: Final[list[tuple[str, str | tuple[str, ...]]]] = [] # mutable-ok: local accumulator - while work: - current_key, current_value, depth = work.pop() + pending_fields: Final[ # mutable-ok: depth-capped stack walks nested JSON into multipart names + list[tuple[str, object, int]] + ] = [ # mutable-ok: depth-capped stack walks nested JSON into multipart names + (key, value, 0) + ] + flat_fields: Final[list[tuple[str, str | tuple[str, ...]]]] = [] # mutable-ok: local accumulator + while pending_fields: + current_key, current_value, depth = pending_fields.pop() if depth > DEFAULT_MAX_RECURSE_DEPTH: raise ValueError("form field nesting exceeds max depth") if isinstance(current_value, Mapping): - work.extend( + pending_fields.extend( (f"{current_key}[{subkey}]", subvalue, depth + 1) for subkey, subvalue in reversed(tuple(current_value.items())) ) @@ -58,16 +66,16 @@ def _flatten_form_data_field(key: str, value: object) -> tuple[tuple[str, str | if all(_is_form_scalar(entry) for entry in current_value): serialized_fields = tuple(field for entry in current_value if (field := _form_field_value(entry))) if serialized_fields: - out.append((current_key, serialized_fields)) + flat_fields.append((current_key, serialized_fields)) continue - work.extend((f"{current_key}[]", entry, depth + 1) for entry in reversed(tuple(current_value))) + pending_fields.extend((f"{current_key}[]", entry, depth + 1) for entry in reversed(tuple(current_value))) continue if current_value is None: continue serialized = _form_field_value(current_value) if serialized: - out.append((current_key, serialized)) - return tuple(out) + flat_fields.append((current_key, serialized)) + return tuple(flat_fields) def flatten_form_field_values(*sources: Mapping[str, object] | None) -> tuple[tuple[str, str | tuple[str, ...]], ...]: diff --git a/litellm/llms/hosted_vllm/videos/transformation.py b/litellm/llms/hosted_vllm/videos/transformation.py index 16f1397be8a..96cbfc3cf70 100644 --- a/litellm/llms/hosted_vllm/videos/transformation.py +++ b/litellm/llms/hosted_vllm/videos/transformation.py @@ -1,34 +1,21 @@ """Video generation for Hosted VLLM (vLLM-Omni OpenAI-compatible /v1/videos).""" -import base64 import json -import mimetypes from collections.abc import Mapping -from dataclasses import dataclass from io import BufferedReader from types import MappingProxyType from typing import Final from urllib.parse import urlparse -from httpx import Response from httpx._types import FileTypes, RequestFiles -from litellm.constants import ( - MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, - MAX_VIDEO_MEDIA_URL_TOTAL_DOWNLOAD_SIZE_MB, - MAX_VIDEO_MEDIA_URLS_PER_REQUEST, -) from litellm.images.utils import ImageEditRequestUtils -from litellm.litellm_core_utils.url_utils import SSRFError, safe_get -from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.litellm_core_utils.url_utils import SSRFError, validate_url from litellm.llms.openai.videos.transformation import OpenAIVideoConfig from litellm.secret_managers.main import get_secret_str from litellm.types.router import GenericLiteLLMParams from litellm.types.videos.main import VideoCreateOptionalRequestParams -_BYTES_PER_MB: Final = 1024 * 1024 -_STREAM_CHUNK_SIZE: Final = 8192 - _EXCLUDED_FORM_KEYS: Final = frozenset( { "model", @@ -79,28 +66,6 @@ _REFERENCE_URL_KEYS: Final = MappingProxyType( ) -@dataclass(slots=True) -class _MediaDownloadBudget: - remaining_urls: int - remaining_bytes: int - max_bytes_per_url: int - - def consume_url_slot(self) -> None: - if self.max_bytes_per_url <= 0: - raise ValueError("remote media URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0)") - if self.remaining_urls < 1: - raise ValueError("too many remote media URL references on one video request") - self.remaining_urls -= 1 - - def max_read_bytes(self) -> int: - return min(self.max_bytes_per_url, self.remaining_bytes) - - def consume_bytes(self, nbytes: int) -> None: - if nbytes > self.remaining_bytes: - raise ValueError("remote media download exceeded the per-request size limit") - self.remaining_bytes -= nbytes - - def _serialize_form_value(value: object) -> str: if isinstance(value, str): return value @@ -120,91 +85,46 @@ def _maybe_json(value: object) -> object: return json.loads(stripped) -def _content_type_from_headers(headers: Mapping[str, object]) -> str: - raw: Final = headers.get("content-type", "application/octet-stream") - if not isinstance(raw, str) or not raw.strip(): - return "application/octet-stream" - return raw.split(";", 1)[0].strip() or "application/octet-stream" - - -def _declared_content_length(headers: Mapping[str, object]) -> int | None: - raw: Final = headers.get("content-length") - if not isinstance(raw, str): - return None - stripped: Final = raw.strip() - if not stripped.isdigit(): - return None - return int(stripped) - - -def _read_capped_body(response: Response, max_bytes: int) -> bytes: - declared: Final = _declared_content_length(response.headers) - if declared is not None and declared > max_bytes: - response.close() - raise ValueError("remote media URL Content-Length exceeds the maximum allowed size") - body: Final = bytearray() # mutable-ok: streaming accumulator with a hard cap - for chunk in response.iter_bytes(chunk_size=_STREAM_CHUNK_SIZE): - body.extend(chunk) - if len(body) > max_bytes: - response.close() - raise ValueError("remote media download exceeded the maximum allowed size") - return bytes(body) - - -def _fetch_url_as_data_url(url: str, client: HTTPHandler, budget: _MediaDownloadBudget) -> str: - budget.consume_url_slot() - response: Final = safe_get(client, url) - response.raise_for_status() - content: Final = _read_capped_body(response, budget.max_read_bytes()) - budget.consume_bytes(len(content)) - encoded: Final = base64.b64encode(content).decode("ascii") - return f"data:{_content_type_from_headers(response.headers)};base64,{encoded}" - - -def _inline_reference_item(url_key: str, item: object, client: HTTPHandler, budget: _MediaDownloadBudget) -> object: - if not isinstance(item, Mapping): - return item - url: Final = item.get(url_key) - if not isinstance(url, str): - return item +def _reject_unsafe_media_url(url: str) -> None: scheme: Final = urlparse(url).scheme.lower() if scheme in ("", "data"): - return item + return if scheme not in ("http", "https"): raise SSRFError(f"URL scheme '{scheme}' is not allowed") - return { # mutable-ok: JSON form field serialized immediately - **item, - url_key: _fetch_url_as_data_url(url, client, budget), - } + validate_url(url) -def _inline_media_reference( - field_name: str, value: object, client: HTTPHandler, budget: _MediaDownloadBudget -) -> object: +def _reject_unsafe_urls_in_item(url_key: str, item: object) -> None: + if not isinstance(item, Mapping): + return + url: Final = item.get(url_key) + if isinstance(url, str): + _reject_unsafe_media_url(url) + + +def _reject_unsafe_media_urls(field_name: str, value: object) -> None: url_key: Final = _REFERENCE_URL_KEYS.get(field_name) if url_key is None: - return value + return parsed: Final = _maybe_json(value) if isinstance(parsed, list): - return tuple(_inline_reference_item(url_key, item, client, budget) for item in parsed) + for item in parsed: + _reject_unsafe_urls_in_item(url_key, item) + return if isinstance(parsed, Mapping): - return _inline_reference_item(url_key, parsed, client, budget) - return value + _reject_unsafe_urls_in_item(url_key, parsed) + + +def _form_value(key: str, value: object) -> str: + _reject_unsafe_media_urls(key, value) + return _serialize_form_value(value) def _input_reference_file(reference: object) -> tuple[str, FileTypes]: + content_type: Final = ImageEditRequestUtils.get_image_content_type(reference) if isinstance(reference, BufferedReader): - reader_name: Final = reference.name - reader_type: Final = mimetypes.guess_type(reader_name)[0] or ImageEditRequestUtils.get_image_content_type( - reference - ) - return ("input_reference", (reader_name, reference, reader_type)) - - fallback_name: Final = getattr(reference, "name", None) or "input_reference.png" - fallback_type: Final = mimetypes.guess_type(str(fallback_name))[0] or ImageEditRequestUtils.get_image_content_type( - reference - ) - return ("input_reference", (str(fallback_name), reference, fallback_type)) + return ("input_reference", (reference.name, reference, content_type)) + return ("input_reference", ("input_reference.png", reference, content_type)) class HostedVLLMVideoConfig(OpenAIVideoConfig): @@ -214,42 +134,6 @@ class HostedVLLMVideoConfig(OpenAIVideoConfig): https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/videos_api/ """ - def __init__( - self, - media_http_client: HTTPHandler | None = None, - *, - max_media_bytes_per_url: int | None = None, - max_media_bytes_per_request: int | None = None, - max_media_urls_per_request: int | None = None, - ) -> None: - super().__init__() - self._media_http_client = media_http_client - self._max_media_bytes_per_url = max_media_bytes_per_url - self._max_media_bytes_per_request = max_media_bytes_per_request - self._max_media_urls_per_request = max_media_urls_per_request - - def _media_budget(self) -> _MediaDownloadBudget: - per_url: Final = ( - self._max_media_bytes_per_url - if self._max_media_bytes_per_url is not None - else int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * _BYTES_PER_MB) - ) - per_request: Final = ( - self._max_media_bytes_per_request - if self._max_media_bytes_per_request is not None - else int(MAX_VIDEO_MEDIA_URL_TOTAL_DOWNLOAD_SIZE_MB * _BYTES_PER_MB) - ) - url_slots: Final = ( - self._max_media_urls_per_request - if self._max_media_urls_per_request is not None - else MAX_VIDEO_MEDIA_URLS_PER_REQUEST - ) - return _MediaDownloadBudget( - remaining_urls=url_slots, - remaining_bytes=per_request, - max_bytes_per_url=per_url, - ) - def get_supported_openai_params(self, model: str) -> list: # mutable-ok: BaseVideoConfig contract return [ # mutable-ok: BaseVideoConfig returns list *super().get_supported_openai_params(model), @@ -307,22 +191,16 @@ class HostedVLLMVideoConfig(OpenAIVideoConfig): litellm_params: GenericLiteLLMParams, headers: dict, # mutable-ok: BaseVideoConfig contract ) -> tuple[dict, RequestFiles, str]: # mutable-ok: BaseVideoConfig contract - media_client: Final = self._media_http_client or HTTPHandler(concurrent_limit=1) - media_budget: Final = self._media_budget() + data: Final = { # mutable-ok: BaseVideoConfig contract returns a data dict + "model": model, + "prompt": prompt, + **{ # mutable-ok: spread remaining Omni form fields into that data dict + key: _form_value(key, value) + for key, value in video_create_optional_request_params.items() + if key not in _EXCLUDED_FORM_KEYS and value is not None + }, + } input_reference: Final = video_create_optional_request_params.get("input_reference") - form_files: Final = tuple( - ( - key, - (None, _serialize_form_value(_inline_media_reference(key, value, media_client, media_budget))), - ) - for key, value in video_create_optional_request_params.items() - if key not in _EXCLUDED_FORM_KEYS and value is not None - ) - reference_files: Final = (_input_reference_file(input_reference),) if input_reference is not None else () - files: Final = ( - ("model", (None, model)), - ("prompt", (None, prompt)), - *form_files, - *reference_files, - ) - return {}, files, api_base # mutable-ok: empty data dict; files carry the multipart fields + if input_reference is None: + return data, (), api_base + return data, (_input_reference_file(input_reference),), api_base diff --git a/tests/test_litellm/llms/hosted_vllm/videos/test_hosted_vllm_video_transformation.py b/tests/test_litellm/llms/hosted_vllm/videos/test_hosted_vllm_video_transformation.py index be02e03833b..eb90430f303 100644 --- a/tests/test_litellm/llms/hosted_vllm/videos/test_hosted_vllm_video_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/videos/test_hosted_vllm_video_transformation.py @@ -1,6 +1,5 @@ """Tests for hosted_vllm video generation (vLLM-Omni /v1/videos).""" -import base64 import json from io import BytesIO @@ -21,10 +20,6 @@ from litellm.types.videos.main import VideoObject from litellm.utils import ProviderConfigManager -def _form_fields(files: list) -> dict[str, str]: - return {name: value[1] for name, value in files if value[0] is None} - - def test_provider_config_registration(): config = ProviderConfigManager.get_provider_video_config( model="hosted_vllm/MiniMax-H3", @@ -109,27 +104,25 @@ def test_transform_video_create_request_uses_multipart_form_fields(): headers={}, ) - assert data == {} assert url == "http://localhost:8091/v1/videos" - assert files - fields = _form_fields(files) - assert fields["model"] == "MiniMax-H3" - assert fields["prompt"] == "three cats march into a bedroom playing tiny brass instruments" - assert fields["width"] == "1280" - assert fields["height"] == "720" - assert fields["fps"] == "24" - assert fields["num_inference_steps"] == "20" - assert fields["flow_shift"] == "12" - assert fields["seed"] == "1101" - assert fields["aspect_ratio"] == "16:9" - assert json.loads(fields["extra_params"]) == extra_params - assert "extra_headers" not in fields + assert files == () + assert data["model"] == "MiniMax-H3" + assert data["prompt"] == "three cats march into a bedroom playing tiny brass instruments" + assert data["width"] == "1280" + assert data["height"] == "720" + assert data["fps"] == "24" + assert data["num_inference_steps"] == "20" + assert data["flow_shift"] == "12" + assert data["seed"] == "1101" + assert data["aspect_ratio"] == "16:9" + assert json.loads(data["extra_params"]) == extra_params + assert "extra_headers" not in data def test_transform_video_create_request_keeps_openai_size_and_seconds(): config = HostedVLLMVideoConfig() - _, files, _ = config.transform_video_create_request( + data, files, _ = config.transform_video_create_request( model="Wan2.2", prompt="a mountain lake at sunrise", api_base="http://localhost:8091/v1/videos", @@ -138,9 +131,9 @@ def test_transform_video_create_request_keeps_openai_size_and_seconds(): headers={}, ) - fields = _form_fields(files) - assert fields["seconds"] == "8" - assert fields["size"] == "1280x720" + assert files == () + assert data["seconds"] == "8" + assert data["size"] == "1280x720" def test_transform_video_create_request_attaches_input_reference_file(): @@ -157,14 +150,12 @@ def test_transform_video_create_request_attaches_input_reference_file(): headers={}, ) - assert data == {} - fields = _form_fields(files) - assert fields["width"] == "832" - assert "input_reference" not in fields + assert data["width"] == "832" + assert "input_reference" not in data reference_parts = [value for name, value in files if name == "input_reference"] assert len(reference_parts) == 1 filename, content, content_type = reference_parts[0] - assert filename == "input.png" + assert filename == "input_reference.png" assert content is reference assert content_type == "image/png" @@ -257,15 +248,9 @@ def test_video_generation_posts_multipart_not_json(): assert request.headers.get("content-type", "").startswith("multipart/form-data") -def test_http_image_reference_is_inlined_as_data_url(): - png_bytes = b"fake-png" - - def handler(request: httpx.Request) -> httpx.Response: - assert request.url.host == "1.1.1.1" - return httpx.Response(200, content=png_bytes, headers={"content-type": "image/png"}) - - config = HostedVLLMVideoConfig(media_http_client=_http_handler_for(handler)) - _, files, _ = config.transform_video_create_request( +def test_http_image_reference_is_forwarded_not_downloaded(): + config = HostedVLLMVideoConfig() + data, files, _ = config.transform_video_create_request( model="MiniMax-H3", prompt="a person singing", api_base="http://localhost:8091/v1/videos", @@ -276,40 +261,15 @@ def test_http_image_reference_is_inlined_as_data_url(): headers={}, ) - payload = json.loads(_form_fields(files)["image_reference"]) - assert payload["image_url"] == "data:image/png;base64," + base64.b64encode(png_bytes).decode("ascii") + assert files == () + payload = json.loads(data["image_reference"]) + assert payload["image_url"] == "http://1.1.1.1/face.png" -def test_http_audio_reference_json_string_is_inlined_as_data_url(): - audio_bytes = b"fake-mp3" - - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(200, content=audio_bytes, headers={"content-type": "audio/mpeg"}) - - config = HostedVLLMVideoConfig(media_http_client=_http_handler_for(handler)) - _, files, _ = config.transform_video_create_request( - model="MiniMax-H3", - prompt="a person singing", - api_base="http://localhost:8091/v1/videos", - video_create_optional_request_params={ - "audio_reference": '{"audio_url": "http://1.1.1.1/speech.mp3"}', - }, - litellm_params=GenericLiteLLMParams(), - headers={}, - ) - - payload = json.loads(_form_fields(files)["audio_reference"]) - assert payload["audio_url"].startswith("data:audio/mpeg;base64,") - assert base64.b64decode(payload["audio_url"].split(",", 1)[1]) == audio_bytes - - -def test_data_url_image_reference_is_not_fetched(): - def handler(request: httpx.Request) -> httpx.Response: - raise AssertionError(f"unexpected fetch of {request.url}") - +def test_data_url_image_reference_is_forwarded(): data_url = "data:image/png;base64,AAAA" - config = HostedVLLMVideoConfig(media_http_client=_http_handler_for(handler)) - _, files, _ = config.transform_video_create_request( + config = HostedVLLMVideoConfig() + data, files, _ = config.transform_video_create_request( model="MiniMax-H3", prompt="a person singing", api_base="http://localhost:8091/v1/videos", @@ -318,7 +278,8 @@ def test_data_url_image_reference_is_not_fetched(): headers={}, ) - assert json.loads(_form_fields(files)["image_reference"])["image_url"] == data_url + assert files == () + assert json.loads(data["image_reference"])["image_url"] == data_url def test_metadata_url_in_image_reference_is_rejected(): @@ -349,60 +310,3 @@ def test_file_scheme_media_reference_is_rejected(): litellm_params=GenericLiteLLMParams(), headers={}, ) - - -def _transform_with_references(config: HostedVLLMVideoConfig, **references: object): - return config.transform_video_create_request( - model="MiniMax-H3", - prompt="a person singing", - api_base="http://localhost:8091/v1/videos", - video_create_optional_request_params=references, - litellm_params=GenericLiteLLMParams(), - headers={}, - ) - - -def test_oversized_content_length_is_rejected_before_body(): - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response( - 200, - content=b"", - headers={"content-type": "video/mp4", "content-length": str(51 * 1024 * 1024)}, - ) - - config = HostedVLLMVideoConfig(media_http_client=_http_handler_for(handler)) - with pytest.raises(ValueError, match="Content-Length"): - _transform_with_references(config, video_reference={"video_url": "http://1.1.1.1/clip.mp4"}) - - -def test_streamed_body_over_per_url_cap_is_rejected(): - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response( - 200, - content=iter((b"1234", b"5678")), - headers={"content-type": "image/png"}, - ) - - config = HostedVLLMVideoConfig( - media_http_client=_http_handler_for(handler), - max_media_bytes_per_url=4, - max_media_bytes_per_request=4, - ) - with pytest.raises(ValueError, match="exceeded the maximum allowed size"): - _transform_with_references(config, image_reference={"image_url": "http://1.1.1.1/face.png"}) - - -def test_too_many_remote_media_urls_are_rejected(): - def handler(request: httpx.Request) -> httpx.Response: - return httpx.Response(200, content=b"ok", headers={"content-type": "image/png"}) - - config = HostedVLLMVideoConfig( - media_http_client=_http_handler_for(handler), - max_media_urls_per_request=1, - ) - with pytest.raises(ValueError, match="too many remote media URL references"): - _transform_with_references( - config, - image_reference={"image_url": "http://1.1.1.1/a.png"}, - audio_reference={"audio_url": "http://1.1.1.1/b.mp3"}, - )