From ad02f1907dce74f5840fd9fb0ab7489e37306e2d Mon Sep 17 00:00:00 2001 From: "moyai-devin-berriai[bot]" <336287033+moyai-devin-berriai[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:50:12 +0000 Subject: [PATCH 1/4] feat(decisions): support Clef-omni audio and video --- docs/decisions/clef-omni-media.md | 46 +++++++ litellm/decisions/main.py | 21 ++- .../llms/base_llm/decisions/transformation.py | 21 ++- .../cloudflare/decisions/transformation.py | 3 + .../llms/openai/decisions/transformation.py | 4 + litellm/types/decisions.py | 33 ++++- model_prices_and_context_window.json | 16 +++ tests/unit/decisions/test_main.py | 124 +++++++++++++++++- 8 files changed, 260 insertions(+), 8 deletions(-) create mode 100644 docs/decisions/clef-omni-media.md diff --git a/docs/decisions/clef-omni-media.md b/docs/decisions/clef-omni-media.md new file mode 100644 index 00000000000..899fd5cf112 --- /dev/null +++ b/docs/decisions/clef-omni-media.md @@ -0,0 +1,46 @@ +# Clef-omni audio and video + +Configure a model with `model: cloudflare/clef-omni` and set `CLOUDFLARE_API_KEY` and `CLOUDFLARE_ACCOUNT_ID`. An explicit `api_base` works as well. The fully qualified `cloudflare/@cf/cloudflare/clef-omni` model name is supported + +Send embedded clips in the top-level `audio` and `videos` lists. Each entry is either a base64 data URL or an object with `content_type` and `base64`. LiteLLM validates the shape, media type and clip count before sending a request. Cloudflare validates the decoded bytes, sizes and durations + +```python +import base64 +from pathlib import Path +import litellm + +result = litellm.decisions( + model="cloudflare/clef-omni", + state="Evaluate these clips.", + questions={ + "speech": {"type": "noul", "instructions": "Is there speech?"}, + "red": {"type": "noul", "instructions": "Does the video show red?"}, + }, + audio=[{ + "content_type": "audio/wav", + "base64": base64.b64encode(Path("clip.wav").read_bytes()).decode(), + }], + videos=["data:video/mp4;base64," + base64.b64encode(Path("clip.mp4").read_bytes()).decode()], +) +print(result.answers) +``` + +The same fields work with `await litellm.adecisions(...)` and the proxy routes `/v1/systemone` and `/systemone` + +For `/v1/decisions` and `/decisions`, use the OpenAI-style `input` and `questions` fields, with the same top-level `audio` and `videos` extensions. These are LiteLLM extensions for Clef-omni, not OpenAI upstream media support. `input_audio` and `input_video` content parts are not introduced by this change + +```json +{ + "model": "your-clef-omni-alias", + "input": "Evaluate the clips.", + "questions": [{"type": "predicate", "name": "speech", "instructions": "Is there speech?"}], + "audio": [{"content_type": "audio/wav", "base64": ""}], + "videos": ["data:video/mp4;base64,"] +} +``` + +Remote URLs are rejected. Empty lists behave like text-only requests. Nonempty audio or video lists are rejected for other Decisions models, including Clef and Clef-flash, rather than silently discarded + +Cloudflare's [Clef-omni documentation](https://developers.cloudflare.com/workers-ai/models/clef-omni/) allows four audio clips (8 MiB and 300 seconds each), two videos (16 MiB and 60 seconds each), and at most 16 MiB of audio and video combined. Videos are sampled at two frames per second. Media consumes input tokens at the model's input price. Cloudflare handles decoding, duration limits and context-window limits + +This change is independent of the image support in PR #45762. It does not add native image handling. diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index f6abb1b2f3a..f6c229a910b 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -19,11 +19,13 @@ from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir from litellm.types.decisions import ( DecisionQuestion, + DecisionsAudio, DecisionsIRRequest, DecisionsIRResponse, DecisionsJSON, DecisionsRequestBody, DecisionsResponse, + DecisionsVideo, OpenAIDecisionInput, OpenAIDecisionQuestion, OpenAIDecisionRequestBody, @@ -92,11 +94,14 @@ def _validate_request( questions: DecisionsQuestions | None, decision_input: OpenAIDecisionInput | None, safety_identifier: str | None, + audio: Sequence[DecisionsAudio | Mapping[str, object]] | None, + videos: Sequence[DecisionsVideo | Mapping[str, object]] | None, ) -> DecisionsRequestFormat: + media: Final = {"audio": audio if audio is not None else (), "videos": videos if videos is not None else ()} if decision_input is None: - return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": state, "questions": questions}) + return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": state, "questions": questions, **media}) return _OPENAI_REQUEST_ADAPTER.validate_python( - {"input": decision_input, "questions": questions, "safety_identifier": safety_identifier} + {"input": decision_input, "questions": questions, "safety_identifier": safety_identifier, **media} ) @@ -147,6 +152,8 @@ def _prepare_call( questions: DecisionsQuestions | None, decision_input: OpenAIDecisionInput | None, safety_identifier: str | None, + audio: Sequence[DecisionsAudio | Mapping[str, object]] | None, + videos: Sequence[DecisionsVideo | Mapping[str, object]] | None, api_key: str | None, api_base: str | None, timeout: float | httpx.Timeout | None, @@ -184,6 +191,8 @@ def _prepare_call( questions=questions, decision_input=decision_input, safety_identifier=request_safety_identifier, + audio=audio, + videos=videos, ) except ValidationError as error: raise litellm.BadRequestError( @@ -314,6 +323,8 @@ async def adecisions( extra_headers: Mapping[str, str] | None = None, input: OpenAIDecisionInput | None = None, safety_identifier: str | None = None, + audio: Sequence[DecisionsAudio | Mapping[str, object]] | None = None, + videos: Sequence[DecisionsVideo | Mapping[str, object]] | None = None, **kwargs: object, ) -> DecisionsResponse | OpenAIDecisionResponse: call: Final = _prepare_call( @@ -322,6 +333,8 @@ async def adecisions( questions=questions, decision_input=input, safety_identifier=safety_identifier, + audio=audio, + videos=videos, api_key=api_key, api_base=api_base, timeout=timeout, @@ -360,6 +373,8 @@ def decisions( extra_headers: Mapping[str, str] | None = None, input: OpenAIDecisionInput | None = None, safety_identifier: str | None = None, + audio: Sequence[DecisionsAudio | Mapping[str, object]] | None = None, + videos: Sequence[DecisionsVideo | Mapping[str, object]] | None = None, **kwargs: object, ) -> DecisionsResponse | OpenAIDecisionResponse: call: Final = _prepare_call( @@ -368,6 +383,8 @@ def decisions( questions=questions, decision_input=input, safety_identifier=safety_identifier, + audio=audio, + videos=videos, api_key=api_key, api_base=api_base, timeout=timeout, diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 724ca23c2ab..33ca895c5e0 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -100,6 +100,8 @@ def _ir_question(name: str, question: DecisionQuestion) -> DecisionsIRQuestion: def systemone_request_to_ir(request: DecisionsRequestBody) -> DecisionsIRRequest: return DecisionsIRRequest( input=DecisionsIRState(state=request.state), + audio=request.audio, + videos=request.videos, questions=tuple(_ir_question(name, question) for name, question in request.questions.items()), ) @@ -166,8 +168,10 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]: def ir_to_systemone_request( - model: str, request: DecisionsIRRequest + model: str, request: DecisionsIRRequest, supports_audio_video: bool = False ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + if (request.audio or request.videos) and not supports_audio_video: + return UnsupportedDecisionsRequest(reason="audio/video input is not supported by this Decisions model") state: Final = _systemone_state(request.input) if isinstance(state, UnsupportedDecisionsRequest): return state @@ -175,6 +179,16 @@ def ir_to_systemone_request( return { "model": model, "state": state, + **( + {"audio": [clip if isinstance(clip, str) else clip.model_dump() for clip in request.audio]} + if request.audio + else {} + ), + **( + {"videos": [clip if isinstance(clip, str) else clip.model_dump() for clip in request.videos]} + if request.videos + else {} + ), "questions": {key: _systemone_question(question) for key, question in keyed_questions}, } @@ -355,13 +369,16 @@ class BaseDecisionsConfig(ABC): ) -> tuple[Mapping[str, str], bytes | None]: return headers, None + def supports_audio_video_input(self, model: str) -> bool: + return False + def transform_decisions_request( self, model: str, request: DecisionsIRRequest, custom_llm_provider: str, ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - return ir_to_systemone_request(self.request_model(model), request) + return ir_to_systemone_request(self.request_model(model), request, self.supports_audio_video_input(model)) def unwrap_response(self, payload: object) -> object: return payload diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index fa8c68ad3a4..6e8a2ebd1ec 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -30,6 +30,9 @@ class CloudflareDecisionsConfig(BaseDecisionsConfig): return model return f"@cf/cloudflare/{model}" + def supports_audio_video_input(self, model: str) -> bool: + return self.canonical_model(model) == "@cf/cloudflare/clef-omni" + def request_model(self, model: str) -> str: return model.rsplit("/", maxsplit=1)[-1] diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py index 317e5abeeb5..a8cd50faafe 100644 --- a/litellm/llms/openai/decisions/transformation.py +++ b/litellm/llms/openai/decisions/transformation.py @@ -88,6 +88,8 @@ def _ir_question(question: OpenAIDecisionQuestion) -> DecisionsIRQuestion: def openai_request_to_ir(request: OpenAIDecisionRequestBody) -> DecisionsIRRequest: return DecisionsIRRequest( input=_ir_input(request.input), + audio=request.audio, + videos=request.videos, questions=tuple(_ir_question(question) for question in request.questions), safety_identifier=request.safety_identifier, ) @@ -164,6 +166,8 @@ def _has_one_option(question: DecisionsIRQuestion) -> bool: def ir_to_openai_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object] | UnsupportedDecisionsRequest: + if request.audio or request.videos: + return UnsupportedDecisionsRequest(reason="audio/video input is not supported by this Decisions provider") if any(_has_one_option(question) for question in request.questions): return _SINGLE_OPTION return { diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index b24690dbd9d..0c030b41a13 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -54,7 +54,32 @@ DecisionQuestionMap: TypeAlias = Annotated[ ] -class DecisionsRequestBody(LiteLLMPydanticObjectBase): +class DecisionsBase64Audio(LiteLLMPydanticObjectBase): + content_type: Annotated[str, Field(pattern=r"^audio/[^;\s]+$")] + base64: Annotated[str, Field(min_length=1)] + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class DecisionsBase64Video(LiteLLMPydanticObjectBase): + content_type: Annotated[str, Field(pattern=r"^video/[^;\s]+$")] + base64: Annotated[str, Field(min_length=1)] + + model_config = ConfigDict(extra="forbid", frozen=True) + + +DecisionsAudio: TypeAlias = Annotated[str, Field(pattern=r"^data:audio/[^;,\s]+;base64,.+$")] | DecisionsBase64Audio +DecisionsVideo: TypeAlias = Annotated[str, Field(pattern=r"^data:video/[^;,\s]+;base64,.+$")] | DecisionsBase64Video + + +class DecisionsMediaBody(LiteLLMPydanticObjectBase): + audio: Annotated[tuple[DecisionsAudio, ...], Field(max_length=4)] = () + videos: Annotated[tuple[DecisionsVideo, ...], Field(max_length=2)] = () + + model_config = ConfigDict(extra="allow", frozen=True) + + +class DecisionsRequestBody(DecisionsMediaBody): state: DecisionsJSON questions: DecisionQuestionMap @@ -70,6 +95,8 @@ class DecisionsCallParams(TypedDict, total=False): model: Required[ReadOnly[str]] state: Required[ReadOnly[DecisionsJSON]] questions: Required[ReadOnly[DecisionQuestionMap]] + audio: ReadOnly[Sequence[DecisionsAudio]] + videos: ReadOnly[Sequence[DecisionsVideo]] api_key: ReadOnly[str | None] api_base: ReadOnly[str | None] timeout: ReadOnly[float | None] @@ -228,7 +255,7 @@ OpenAIDecisionQuestion: TypeAlias = Annotated[ ] -class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase): +class OpenAIDecisionRequestBody(DecisionsMediaBody): input: OpenAIDecisionInput questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=MAX_DECISION_QUESTIONS)] safety_identifier: str | None = None @@ -380,6 +407,8 @@ class DecisionsIRRequest: input: DecisionsIRState | DecisionsIRMessages questions: tuple[DecisionsIRQuestion, ...] safety_identifier: str | None = None + audio: tuple[DecisionsAudio, ...] = () + videos: tuple[DecisionsVideo, ...] = () @dataclass(frozen=True, slots=True) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index ce38a4689c3..5ad29aa0988 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15964,6 +15964,22 @@ "prompt_cache_min_tokens": 1024, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "cloudflare/clef-omni": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 64000, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/" + }, + "cloudflare/@cf/cloudflare/clef-omni": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 64000, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/" + }, "cloudflare/clef": { "input_cost_per_token": 2.4e-07, "litellm_provider": "cloudflare", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index ebe0eedea7f..efe460a8152 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -3,8 +3,8 @@ from __future__ import annotations import asyncio import json import threading -from concurrent.futures import wait from collections.abc import Mapping +from concurrent.futures import wait from types import MappingProxyType from typing import Final @@ -16,8 +16,8 @@ from botocore.awsrequest import AWSRequest from botocore.credentials import Credentials import litellm -from litellm.cost_calculator import get_response_cost_from_hidden_params from litellm.constants import AWS_SIGNING_MAX_THREADS +from litellm.cost_calculator import get_response_cost_from_hidden_params from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.llms.bedrock.base_aws_llm import AWS_SIGNING_EXECUTOR @@ -1680,3 +1680,123 @@ async def test_azure_ai_decision_requires_api_base_before_http( ) assert len(respx_mock.calls) == 0 + + +@pytest.mark.parametrize("async_mode", (False, True)) +@pytest.mark.parametrize("openai_format", (False, True)) +@pytest.mark.parametrize("model", ("cloudflare/clef-omni", "cloudflare/@cf/cloudflare/clef-omni")) +@pytest.mark.asyncio +async def test_clef_omni_media_reaches_upstream_in_both_formats( + respx_mock: respx.MockRouter, async_mode: bool, openai_format: bool, model: str +) -> None: + audio: Final = ["data:audio/wav;base64,YQ==", {"content_type": "audio/mpeg", "base64": "Yg=="}] + videos: Final = ["data:video/mp4;base64,Yw==", {"content_type": "video/webm", "base64": "ZA=="}] + route: Final = respx_mock.post("https://example.com/ai/run/@cf/cloudflare/clef-omni").respond( + 200, + json={ + "success": True, + "result": { + "model": "clef-omni", + "answers": {"heard": {"type": "noul", "noul": 0.9}}, + "usage": {"input_tokens": 100, "output_tokens": 0}, + }, + }, + ) + fields: Final = ( + { + "input": "Evaluate the clips", + "questions": [{"type": "predicate", "name": "heard", "instructions": "Is there speech?"}], + } + if openai_format + else { + "state": "Evaluate the clips", + "questions": {"heard": {"type": "noul", "instructions": "Is there speech?"}}, + } + ) + response: Final = ( + await litellm.adecisions( + model=model, api_base="https://example.com", api_key="test", audio=audio, videos=videos, **fields + ) + if async_mode + else litellm.decisions( + model=model, api_base="https://example.com", api_key="test", audio=audio, videos=videos, **fields + ) + ) + assert json.loads(route.calls[0].request.content) == { + "model": "clef-omni", + "state": "Evaluate the clips", + "questions": {"heard": {"type": "noul", "instructions": "Is there speech?"}}, + "audio": audio, + "videos": videos, + } + assert response.usage is not None + assert response.usage.input_tokens == 100 + assert len(route.calls) == 1 + + +@pytest.mark.parametrize( + "model", ("cloudflare/clef", "cloudflare/clef-flash", "typesafe/jev-latest", "openai/decision") +) +@pytest.mark.parametrize( + "media", ({"audio": ["data:audio/wav;base64,YQ=="]}, {"videos": ["data:video/mp4;base64,YQ=="]}) +) +@pytest.mark.asyncio +async def test_decisions_rejects_media_for_other_models_before_http( + respx_mock: respx.MockRouter, model: str, media: Mapping[str, object] +) -> None: + with pytest.raises(litellm.BadRequestError, match="audio/video input is not supported"): + await litellm.adecisions( + model=model, + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + **media, + ) + assert len(respx_mock.calls) == 0 + + +@pytest.mark.parametrize( + "media", + ( + {"audio": ["https://example.com/clip.wav"]}, + {"videos": ["https://example.com/clip.mp4"]}, + {"audio": [{"content_type": "audio/wav"}]}, + {"videos": [{"content_type": "video/mp4"}]}, + {"audio": [{"content_type": "video/mp4", "base64": "YQ=="}]}, + {"videos": ["data:audio/wav;base64,YQ=="]}, + {"audio": ["data:audio/wav;base64,YQ=="] * 5}, + {"videos": ["data:video/mp4;base64,YQ=="] * 3}, + ), +) +@pytest.mark.asyncio +async def test_decisions_validates_media_before_http(respx_mock: respx.MockRouter, media: Mapping[str, object]) -> None: + with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"): + await litellm.adecisions( + model="cloudflare/clef-omni", + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + **media, + ) + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_empty_media_keeps_text_only_request_unchanged(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://example.com/v1/systemone").respond(200, json=dict(_RESPONSE)) + await litellm.adecisions( + model="typesafe/jev-latest", + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + audio=[], + videos=[], + ) + assert json.loads(route.calls[0].request.content) == { + "model": "jev-latest", + "state": "test", + "questions": _QUESTIONS, + } From 5861eb9a2185c8e9bbb3879682baa9cb3f569d11 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:48:35 -0700 Subject: [PATCH 2/4] fix(decisions): accept Clef-omni media types with parameters and cover the proxy routes The clip patterns now mirror Cloudflare's input schema, so a data URL or content_type carrying parameters like audio/webm;codecs=opus is accepted. The four proxy routes get media tests of their own, the backup cost map is synced with the new Clef-omni rows, and the docs page moves to the docs repo as a follow-up. --- docs/decisions/clef-omni-media.md | 46 -------- ...odel_prices_and_context_window_backup.json | 16 +++ tests/unit/decisions/test_main.py | 8 +- .../decisions_endpoints/test_endpoints.py | 108 ++++++++++++++++++ 4 files changed, 130 insertions(+), 48 deletions(-) delete mode 100644 docs/decisions/clef-omni-media.md diff --git a/docs/decisions/clef-omni-media.md b/docs/decisions/clef-omni-media.md deleted file mode 100644 index 899fd5cf112..00000000000 --- a/docs/decisions/clef-omni-media.md +++ /dev/null @@ -1,46 +0,0 @@ -# Clef-omni audio and video - -Configure a model with `model: cloudflare/clef-omni` and set `CLOUDFLARE_API_KEY` and `CLOUDFLARE_ACCOUNT_ID`. An explicit `api_base` works as well. The fully qualified `cloudflare/@cf/cloudflare/clef-omni` model name is supported - -Send embedded clips in the top-level `audio` and `videos` lists. Each entry is either a base64 data URL or an object with `content_type` and `base64`. LiteLLM validates the shape, media type and clip count before sending a request. Cloudflare validates the decoded bytes, sizes and durations - -```python -import base64 -from pathlib import Path -import litellm - -result = litellm.decisions( - model="cloudflare/clef-omni", - state="Evaluate these clips.", - questions={ - "speech": {"type": "noul", "instructions": "Is there speech?"}, - "red": {"type": "noul", "instructions": "Does the video show red?"}, - }, - audio=[{ - "content_type": "audio/wav", - "base64": base64.b64encode(Path("clip.wav").read_bytes()).decode(), - }], - videos=["data:video/mp4;base64," + base64.b64encode(Path("clip.mp4").read_bytes()).decode()], -) -print(result.answers) -``` - -The same fields work with `await litellm.adecisions(...)` and the proxy routes `/v1/systemone` and `/systemone` - -For `/v1/decisions` and `/decisions`, use the OpenAI-style `input` and `questions` fields, with the same top-level `audio` and `videos` extensions. These are LiteLLM extensions for Clef-omni, not OpenAI upstream media support. `input_audio` and `input_video` content parts are not introduced by this change - -```json -{ - "model": "your-clef-omni-alias", - "input": "Evaluate the clips.", - "questions": [{"type": "predicate", "name": "speech", "instructions": "Is there speech?"}], - "audio": [{"content_type": "audio/wav", "base64": ""}], - "videos": ["data:video/mp4;base64,"] -} -``` - -Remote URLs are rejected. Empty lists behave like text-only requests. Nonempty audio or video lists are rejected for other Decisions models, including Clef and Clef-flash, rather than silently discarded - -Cloudflare's [Clef-omni documentation](https://developers.cloudflare.com/workers-ai/models/clef-omni/) allows four audio clips (8 MiB and 300 seconds each), two videos (16 MiB and 60 seconds each), and at most 16 MiB of audio and video combined. Videos are sampled at two frames per second. Media consumes input tokens at the model's input price. Cloudflare handles decoding, duration limits and context-window limits - -This change is independent of the image support in PR #45762. It does not add native image handling. diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 20558be3422..e52d6ea70af 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15982,6 +15982,22 @@ "prompt_cache_min_tokens": 1024, "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, + "cloudflare/clef-omni": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 64000, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/" + }, + "cloudflare/@cf/cloudflare/clef-omni": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "cloudflare", + "max_input_tokens": 64000, + "mode": "evaluation", + "output_cost_per_token": 0.0, + "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/" + }, "cloudflare/clef": { "input_cost_per_token": 2.4e-07, "litellm_provider": "cloudflare", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index efe460a8152..8d286f949a3 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -1689,8 +1689,12 @@ async def test_azure_ai_decision_requires_api_base_before_http( async def test_clef_omni_media_reaches_upstream_in_both_formats( respx_mock: respx.MockRouter, async_mode: bool, openai_format: bool, model: str ) -> None: - audio: Final = ["data:audio/wav;base64,YQ==", {"content_type": "audio/mpeg", "base64": "Yg=="}] - videos: Final = ["data:video/mp4;base64,Yw==", {"content_type": "video/webm", "base64": "ZA=="}] + audio: Final = [ + "data:audio/wav;base64,YQ==", + {"content_type": "audio/mpeg", "base64": "Yg=="}, + "data:audio/webm;codecs=opus;base64,ZQ==", + ] + videos: Final = ["data:video/mp4;base64,Yw==", {"content_type": "video/webm;codecs=vp9", "base64": "ZA=="}] route: Final = respx_mock.post("https://example.com/ai/run/@cf/cloudflare/clef-omni").respond( 200, json={ diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 87b2b64ad8d..0c100fae822 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -577,6 +577,114 @@ def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_ assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) +_CLEF_OMNI_MEDIA: Final[Mapping[str, object]] = { + "audio": ["data:audio/wav;base64,YQ==", {"content_type": "audio/webm;codecs=opus", "base64": "Yg=="}], + "videos": ["data:video/mp4;base64,Yw=="], +} +_CLEF_OMNI_RESPONSE: Final[Mapping[str, object]] = { + "success": True, + "result": { + "model": "clef-omni", + "answers": {"heard": {"type": "noul", "noul": 0.9}}, + "usage": {"input_tokens": 100, "output_tokens": 0}, + }, +} +_CLEF_OMNI_UPSTREAM: Final = "https://cf.example/ai/run/@cf/cloudflare/clef-omni" + + +def _route_to_clef_omni(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "clef-omni", + "litellm_params": { + "model": "cloudflare/clef-omni", + "api_base": "https://cf.example", + "api_key": "cf-key", + }, + }, + { + "model_name": "decider", + "litellm_params": {"model": "perplexity/pplx-decider-v1-27b", "api_key": "k"}, + }, + ] + ), + ) + + +def _clef_omni_request(endpoint: str) -> Mapping[str, object]: + if endpoint.endswith("systemone"): + return { + "model": "clef-omni", + "state": "Evaluate the clips", + "questions": {"heard": {"type": "noul", "instructions": "Is there speech?"}}, + **_CLEF_OMNI_MEDIA, + } + return { + "model": "clef-omni", + "input": "Evaluate the clips", + "questions": [{"type": "predicate", "name": "heard", "instructions": "Is there speech?"}], + **_CLEF_OMNI_MEDIA, + } + + +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone", "/v1/decisions", "/decisions")) +def test_proxy_forwards_clef_omni_media_on_every_decisions_route( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + endpoint: str, +) -> None: + _route_to_clef_omni(monkeypatch) + upstream: Final = respx_mock.post(_CLEF_OMNI_UPSTREAM).respond(json=_CLEF_OMNI_RESPONSE) + + response: Final = client.post(endpoint, json=_clef_omni_request(endpoint)) + + assert response.status_code == 200, response.text + assert response.json()["answers"] == ( + {"heard": {"type": "noul", "noul": 0.9}} + if endpoint.endswith("systemone") + else [{"type": "predicate", "name": "heard", "probability": 0.9}] + ) + assert json.loads(upstream.calls[0].request.content) == { + "model": "clef-omni", + "state": "Evaluate the clips", + "questions": {"heard": {"type": "noul", "instructions": "Is there speech?"}}, + **_CLEF_OMNI_MEDIA, + } + + +@pytest.mark.parametrize( + ("endpoint", "request_body"), + ( + ("/v1/systemone", {**_REQUEST, **_CLEF_OMNI_MEDIA}), + ("/v1/decisions", {**_clef_omni_request("/v1/decisions"), "audio": [{"content_type": "audio/wav"}]}), + ("/systemone", {**_clef_omni_request("/systemone"), "audio": ["data:image/png;base64,AA=="]}), + ("/decisions", {**_clef_omni_request("/decisions"), "videos": ["data:video/mp4;base64,Yw=="] * 3}), + ), + ids=("model_without_media_support", "clip_missing_base64", "audio_slot_holding_an_image", "too_many_videos"), +) +def test_proxy_refuses_decisions_media_it_cannot_send( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + endpoint: str, + request_body: Mapping[str, object], +) -> None: + _route_to_clef_omni(monkeypatch) + clef_upstream: Final = respx_mock.post(_CLEF_OMNI_UPSTREAM).respond(json=_CLEF_OMNI_RESPONSE) + perplexity_upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post(endpoint, json=request_body) + + assert response.status_code == 400, response.text + assert not clef_upstream.called + assert not perplexity_upstream.called + + @pytest.mark.parametrize("safety_identifier", (7, ["end-user-1"]), ids=("numeric", "list")) @pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params")) def test_a_non_string_safety_identifier_is_refused_unless_the_deployment_drops_params( From cff22b568cbcf8950cd348996503a46218102ab1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:09:13 -0700 Subject: [PATCH 3/4] fix(decisions): refuse parameterized media-type data URLs and refresh the OpenAPI snapshot Cloudflare answers 422 to a data URL whose media type carries parameters even though its schema only requires the data:audio/ prefix, so LiteLLM now refuses those with a 400 before the call. The object form keeps passing parameters through, which Cloudflare accepts. The lazy OpenAPI snapshot and the dashboard API types are regenerated for the new audio and videos fields --- litellm/proxy/_lazy_openapi_snapshot.json | 304 ++++++++++++++++++ litellm/types/decisions.py | 4 +- tests/unit/decisions/test_main.py | 4 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 136 ++++++++ 4 files changed, 445 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 534796032c6..e441cda5374 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -10040,6 +10040,44 @@ "schema": { "additionalProperties": true, "properties": { + "audio": { + "default": [], + "description": "Clef-omni only. Up to 4 embedded audio clips, each a base64 data URL or {content_type, base64}", + "items": { + "anyOf": [ + { + "pattern": "^data:audio/[^;,\\s]+;base64,.+$", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "base64": { + "description": "Base64-encoded audio bytes", + "minLength": 1, + "title": "Base64", + "type": "string" + }, + "content_type": { + "description": "An audio/* media type, such as audio/wav", + "pattern": "^audio/\\S+$", + "title": "Content Type", + "type": "string" + } + }, + "required": [ + "content_type", + "base64" + ], + "title": "DecisionsBase64Audio", + "type": "object" + } + ] + }, + "maxItems": 4, + "title": "Audio", + "type": "array" + }, "input": { "anyOf": [ { @@ -10345,6 +10383,44 @@ ], "description": "A stable id for your end user. Sent to OpenAI and dropped for other providers", "title": "Safety Identifier" + }, + "videos": { + "default": [], + "description": "Clef-omni only. Up to 2 embedded videos, each a base64 data URL or {content_type, base64}", + "items": { + "anyOf": [ + { + "pattern": "^data:video/[^;,\\s]+;base64,.+$", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "base64": { + "description": "Base64-encoded video bytes", + "minLength": 1, + "title": "Base64", + "type": "string" + }, + "content_type": { + "description": "A video/* media type, such as video/mp4", + "pattern": "^video/\\S+$", + "title": "Content Type", + "type": "string" + } + }, + "required": [ + "content_type", + "base64" + ], + "title": "DecisionsBase64Video", + "type": "object" + } + ] + }, + "maxItems": 2, + "title": "Videos", + "type": "array" } }, "required": [ @@ -10419,6 +10495,44 @@ "schema": { "additionalProperties": true, "properties": { + "audio": { + "default": [], + "description": "Clef-omni only. Up to 4 embedded audio clips, each a base64 data URL or {content_type, base64}", + "items": { + "anyOf": [ + { + "pattern": "^data:audio/[^;,\\s]+;base64,.+$", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "base64": { + "description": "Base64-encoded audio bytes", + "minLength": 1, + "title": "Base64", + "type": "string" + }, + "content_type": { + "description": "An audio/* media type, such as audio/wav", + "pattern": "^audio/\\S+$", + "title": "Content Type", + "type": "string" + } + }, + "required": [ + "content_type", + "base64" + ], + "title": "DecisionsBase64Audio", + "type": "object" + } + ] + }, + "maxItems": 4, + "title": "Audio", + "type": "array" + }, "model": { "description": "A decision model from the proxy model_list", "title": "Model", @@ -10647,6 +10761,44 @@ ], "description": "The thing being judged, such as a support ticket, a document or a chat transcript", "title": "State" + }, + "videos": { + "default": [], + "description": "Clef-omni only. Up to 2 embedded videos, each a base64 data URL or {content_type, base64}", + "items": { + "anyOf": [ + { + "pattern": "^data:video/[^;,\\s]+;base64,.+$", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "base64": { + "description": "Base64-encoded video bytes", + "minLength": 1, + "title": "Base64", + "type": "string" + }, + "content_type": { + "description": "A video/* media type, such as video/mp4", + "pattern": "^video/\\S+$", + "title": "Content Type", + "type": "string" + } + }, + "required": [ + "content_type", + "base64" + ], + "title": "DecisionsBase64Video", + "type": "object" + } + ] + }, + "maxItems": 2, + "title": "Videos", + "type": "array" } }, "required": [ @@ -10736,6 +10888,44 @@ "schema": { "additionalProperties": true, "properties": { + "audio": { + "default": [], + "description": "Clef-omni only. Up to 4 embedded audio clips, each a base64 data URL or {content_type, base64}", + "items": { + "anyOf": [ + { + "pattern": "^data:audio/[^;,\\s]+;base64,.+$", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "base64": { + "description": "Base64-encoded audio bytes", + "minLength": 1, + "title": "Base64", + "type": "string" + }, + "content_type": { + "description": "An audio/* media type, such as audio/wav", + "pattern": "^audio/\\S+$", + "title": "Content Type", + "type": "string" + } + }, + "required": [ + "content_type", + "base64" + ], + "title": "DecisionsBase64Audio", + "type": "object" + } + ] + }, + "maxItems": 4, + "title": "Audio", + "type": "array" + }, "input": { "anyOf": [ { @@ -11041,6 +11231,44 @@ ], "description": "A stable id for your end user. Sent to OpenAI and dropped for other providers", "title": "Safety Identifier" + }, + "videos": { + "default": [], + "description": "Clef-omni only. Up to 2 embedded videos, each a base64 data URL or {content_type, base64}", + "items": { + "anyOf": [ + { + "pattern": "^data:video/[^;,\\s]+;base64,.+$", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "base64": { + "description": "Base64-encoded video bytes", + "minLength": 1, + "title": "Base64", + "type": "string" + }, + "content_type": { + "description": "A video/* media type, such as video/mp4", + "pattern": "^video/\\S+$", + "title": "Content Type", + "type": "string" + } + }, + "required": [ + "content_type", + "base64" + ], + "title": "DecisionsBase64Video", + "type": "object" + } + ] + }, + "maxItems": 2, + "title": "Videos", + "type": "array" } }, "required": [ @@ -11115,6 +11343,44 @@ "schema": { "additionalProperties": true, "properties": { + "audio": { + "default": [], + "description": "Clef-omni only. Up to 4 embedded audio clips, each a base64 data URL or {content_type, base64}", + "items": { + "anyOf": [ + { + "pattern": "^data:audio/[^;,\\s]+;base64,.+$", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "base64": { + "description": "Base64-encoded audio bytes", + "minLength": 1, + "title": "Base64", + "type": "string" + }, + "content_type": { + "description": "An audio/* media type, such as audio/wav", + "pattern": "^audio/\\S+$", + "title": "Content Type", + "type": "string" + } + }, + "required": [ + "content_type", + "base64" + ], + "title": "DecisionsBase64Audio", + "type": "object" + } + ] + }, + "maxItems": 4, + "title": "Audio", + "type": "array" + }, "model": { "description": "A decision model from the proxy model_list", "title": "Model", @@ -11343,6 +11609,44 @@ ], "description": "The thing being judged, such as a support ticket, a document or a chat transcript", "title": "State" + }, + "videos": { + "default": [], + "description": "Clef-omni only. Up to 2 embedded videos, each a base64 data URL or {content_type, base64}", + "items": { + "anyOf": [ + { + "pattern": "^data:video/[^;,\\s]+;base64,.+$", + "type": "string" + }, + { + "additionalProperties": false, + "properties": { + "base64": { + "description": "Base64-encoded video bytes", + "minLength": 1, + "title": "Base64", + "type": "string" + }, + "content_type": { + "description": "A video/* media type, such as video/mp4", + "pattern": "^video/\\S+$", + "title": "Content Type", + "type": "string" + } + }, + "required": [ + "content_type", + "base64" + ], + "title": "DecisionsBase64Video", + "type": "object" + } + ] + }, + "maxItems": 2, + "title": "Videos", + "type": "array" } }, "required": [ diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 2b58b6dcc5e..fc6fe977fa3 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -81,8 +81,8 @@ class DecisionsBase64Video(LiteLLMPydanticObjectBase): model_config = ConfigDict(extra="forbid", frozen=True) -DecisionsAudio: TypeAlias = Annotated[str, Field(pattern=r"^data:audio/[^,]+;base64,.+$")] | DecisionsBase64Audio -DecisionsVideo: TypeAlias = Annotated[str, Field(pattern=r"^data:video/[^,]+;base64,.+$")] | DecisionsBase64Video +DecisionsAudio: TypeAlias = Annotated[str, Field(pattern=r"^data:audio/[^;,\s]+;base64,.+$")] | DecisionsBase64Audio +DecisionsVideo: TypeAlias = Annotated[str, Field(pattern=r"^data:video/[^;,\s]+;base64,.+$")] | DecisionsBase64Video class DecisionsMediaBody(LiteLLMPydanticObjectBase): diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 8d286f949a3..b40aed8e40a 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -1692,7 +1692,7 @@ async def test_clef_omni_media_reaches_upstream_in_both_formats( audio: Final = [ "data:audio/wav;base64,YQ==", {"content_type": "audio/mpeg", "base64": "Yg=="}, - "data:audio/webm;codecs=opus;base64,ZQ==", + {"content_type": "audio/webm;codecs=opus", "base64": "ZQ=="}, ] videos: Final = ["data:video/mp4;base64,Yw==", {"content_type": "video/webm;codecs=vp9", "base64": "ZA=="}] route: Final = respx_mock.post("https://example.com/ai/run/@cf/cloudflare/clef-omni").respond( @@ -1769,6 +1769,8 @@ async def test_decisions_rejects_media_for_other_models_before_http( {"videos": [{"content_type": "video/mp4"}]}, {"audio": [{"content_type": "video/mp4", "base64": "YQ=="}]}, {"videos": ["data:audio/wav;base64,YQ=="]}, + {"audio": ["data:audio/webm;codecs=opus;base64,YQ=="]}, + {"videos": ["data:video/webm;codecs=vp9;base64,YQ=="]}, {"audio": ["data:audio/wav;base64,YQ=="] * 5}, {"videos": ["data:video/mp4;base64,YQ=="] * 3}, ), diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 05e80d26e09..ee521c11d07 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -56851,6 +56851,23 @@ export interface operations { * } */ "application/json": { + /** + * Audio + * @description Clef-omni only. Up to 4 embedded audio clips, each a base64 data URL or {content_type, base64} + * @default [] + */ + audio?: (string | { + /** + * Base64 + * @description Base64-encoded audio bytes + */ + base64: string; + /** + * Content Type + * @description An audio/* media type, such as audio/wav + */ + content_type: string; + })[]; /** * Input * @description The text being judged, or user messages whose content mixes input_text and input_image parts @@ -56972,6 +56989,23 @@ export interface operations { * @description A stable id for your end user. Sent to OpenAI and dropped for other providers */ safety_identifier?: string | null; + /** + * Videos + * @description Clef-omni only. Up to 2 embedded videos, each a base64 data URL or {content_type, base64} + * @default [] + */ + videos?: (string | { + /** + * Base64 + * @description Base64-encoded video bytes + */ + base64: string; + /** + * Content Type + * @description A video/* media type, such as video/mp4 + */ + content_type: string; + })[]; } & { [key: string]: unknown; }; @@ -71125,6 +71159,23 @@ export interface operations { * } */ "application/json": { + /** + * Audio + * @description Clef-omni only. Up to 4 embedded audio clips, each a base64 data URL or {content_type, base64} + * @default [] + */ + audio?: (string | { + /** + * Base64 + * @description Base64-encoded audio bytes + */ + base64: string; + /** + * Content Type + * @description An audio/* media type, such as audio/wav + */ + content_type: string; + })[]; /** * Model * @description A decision model from the proxy model_list @@ -71214,6 +71265,23 @@ export interface operations { state: string | { [key: string]: unknown; } | unknown[]; + /** + * Videos + * @description Clef-omni only. Up to 2 embedded videos, each a base64 data URL or {content_type, base64} + * @default [] + */ + videos?: (string | { + /** + * Base64 + * @description Base64-encoded video bytes + */ + base64: string; + /** + * Content Type + * @description A video/* media type, such as video/mp4 + */ + content_type: string; + })[]; } & { [key: string]: unknown; }; @@ -76432,6 +76500,23 @@ export interface operations { * } */ "application/json": { + /** + * Audio + * @description Clef-omni only. Up to 4 embedded audio clips, each a base64 data URL or {content_type, base64} + * @default [] + */ + audio?: (string | { + /** + * Base64 + * @description Base64-encoded audio bytes + */ + base64: string; + /** + * Content Type + * @description An audio/* media type, such as audio/wav + */ + content_type: string; + })[]; /** * Input * @description The text being judged, or user messages whose content mixes input_text and input_image parts @@ -76553,6 +76638,23 @@ export interface operations { * @description A stable id for your end user. Sent to OpenAI and dropped for other providers */ safety_identifier?: string | null; + /** + * Videos + * @description Clef-omni only. Up to 2 embedded videos, each a base64 data URL or {content_type, base64} + * @default [] + */ + videos?: (string | { + /** + * Base64 + * @description Base64-encoded video bytes + */ + base64: string; + /** + * Content Type + * @description A video/* media type, such as video/mp4 + */ + content_type: string; + })[]; } & { [key: string]: unknown; }; @@ -79762,6 +79864,23 @@ export interface operations { * } */ "application/json": { + /** + * Audio + * @description Clef-omni only. Up to 4 embedded audio clips, each a base64 data URL or {content_type, base64} + * @default [] + */ + audio?: (string | { + /** + * Base64 + * @description Base64-encoded audio bytes + */ + base64: string; + /** + * Content Type + * @description An audio/* media type, such as audio/wav + */ + content_type: string; + })[]; /** * Model * @description A decision model from the proxy model_list @@ -79851,6 +79970,23 @@ export interface operations { state: string | { [key: string]: unknown; } | unknown[]; + /** + * Videos + * @description Clef-omni only. Up to 2 embedded videos, each a base64 data URL or {content_type, base64} + * @default [] + */ + videos?: (string | { + /** + * Base64 + * @description Base64-encoded video bytes + */ + base64: string; + /** + * Content Type + * @description A video/* media type, such as video/mp4 + */ + content_type: string; + })[]; } & { [key: string]: unknown; }; From b9746ba51373dd8d753d61832344acfe9cc998f1 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 11 Oct 2026 11:07:56 -0700 Subject: [PATCH 4/4] fix(decisions): gate media by cost-map flags, treat null clips as none, drop unsupported clips before validation --- litellm/__init__.py | 1 + litellm/_lazy_imports_registry.py | 2 + litellm/decisions/call.py | 65 ++++++++------ .../llms/base_llm/decisions/transformation.py | 23 +++-- .../cloudflare/decisions/transformation.py | 3 - ...odel_prices_and_context_window_backup.json | 8 +- .../proxy/decisions_endpoints/endpoints.py | 5 +- litellm/types/decisions.py | 7 +- litellm/types/utils.py | 1 + litellm/utils.py | 6 ++ model_prices_and_context_window.json | 8 +- .../test_cloudflare_clef_omni_media_wire.py | 14 +-- tests/unit/decisions/test_main.py | 87 +++++++++++++++++-- .../test_base_decisions_transformation.py | 20 +++++ .../decisions_endpoints/test_endpoints.py | 67 ++++++++++++++ tests/unit/test_utils.py | 20 +++++ 16 files changed, 281 insertions(+), 56 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index a611b90bc37..ceae8388eee 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -2276,6 +2276,7 @@ if TYPE_CHECKING: supports_parallel_function_calling: Callable[..., bool] supports_vision: Callable[..., bool] supports_audio_input: Callable[..., bool] + supports_video_input: Callable[..., bool] supports_audio_output: Callable[..., bool] supports_system_messages: Callable[..., bool] supports_reasoning: Callable[..., bool] diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index ce69de3b240..0e7cdd115b3 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -36,6 +36,7 @@ UTILS_NAMES: Final = ( "supports_parallel_function_calling", "supports_vision", "supports_audio_input", + "supports_video_input", "supports_audio_output", "supports_system_messages", "supports_reasoning", @@ -481,6 +482,7 @@ _UTILS_IMPORT_MAP: Final = { ), "supports_vision": (".utils", "supports_vision"), "supports_audio_input": (".utils", "supports_audio_input"), + "supports_video_input": (".utils", "supports_video_input"), "supports_audio_output": (".utils", "supports_audio_output"), "supports_system_messages": (".utils", "supports_system_messages"), "supports_reasoning": (".utils", "supports_reasoning"), diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index a5c265102fd..6d014bebee3 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -49,10 +49,6 @@ _SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter( _HANDLER: Final = BaseLLMHTTPHandler() -def _media_fields(audio: DecisionsAudioClips | None, videos: DecisionsVideoClips | None) -> Mapping[str, object]: - return {"audio": audio if audio is not None else (), "videos": videos if videos is not None else ()} - - @dataclass(frozen=True, slots=True) class SystemOneRequestFields: state: DecisionsJSON | None @@ -62,7 +58,7 @@ class SystemOneRequestFields: def validate(self, safety_identifier: str | None) -> DecisionsRequestBody: return _SYSTEMONE_REQUEST_ADAPTER.validate_python( - {"state": self.state, "questions": self.questions, **_media_fields(self.audio, self.videos)} + {"state": self.state, "questions": self.questions, "audio": self.audio, "videos": self.videos} ) @@ -79,7 +75,8 @@ class OpenAIDecisionRequestFields: "input": self.input, "questions": self.questions, "safety_identifier": safety_identifier, - **_media_fields(self.audio, self.videos), + "audio": self.audio, + "videos": self.videos, } ) @@ -141,28 +138,47 @@ def _drops_params(kwargs: Mapping[str, object]) -> bool: return litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True -def _request_safety_identifier(safety_identifier: object, kwargs: Mapping[str, object]) -> str | None: - if isinstance(safety_identifier, str) or not _drops_params(kwargs): +def _request_safety_identifier( + safety_identifier: object, provider_config: BaseDecisionsConfig, kwargs: Mapping[str, object] +) -> str | None: + if not _drops_params(kwargs): return _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier) + if isinstance(safety_identifier, str) and provider_config.supports_safety_identifier: + return safety_identifier return None +def _fields_without_unsupported_media( + fields: RequestFields, + provider_config: BaseDecisionsConfig, + model: str, + custom_llm_provider: str, + kwargs: Mapping[str, object], +) -> RequestFields: + if not _drops_params(kwargs): + return fields + return replace( + fields, + audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None, + videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None, + ) + + @dataclass(frozen=True, slots=True) class _UnsupportedParams: names: tuple[str, ...] def _unsupported_param_names( - ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, model: str + ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str ) -> tuple[str, ...]: - takes_media: Final = provider_config.supports_audio_video_input(model) checks: Final = ( ( "safety_identifier", ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier, ), - ("audio", bool(ir_request.audio) and not takes_media), - ("videos", bool(ir_request.videos) and not takes_media), + ("audio", bool(ir_request.audio) and not provider_config.supports_audio_input(model, custom_llm_provider)), + ("videos", bool(ir_request.videos) and not provider_config.supports_video_input(model, custom_llm_provider)), ) return tuple(name for name, unsupported in checks if unsupported) @@ -173,20 +189,13 @@ def _provider_ir_request( safety_identifier: str | None, provider_config: BaseDecisionsConfig, model: str, - kwargs: Mapping[str, object], + custom_llm_provider: str, ) -> DecisionsIRRequest | _UnsupportedParams: ir_request: Final = _ir_request(request, safety_identifier) - unsupported: Final = _unsupported_param_names(ir_request, provider_config, model) - if not unsupported: - return ir_request - if _drops_params(kwargs): - return replace( - ir_request, - safety_identifier=None if "safety_identifier" in unsupported else ir_request.safety_identifier, - audio=() if "audio" in unsupported else ir_request.audio, - videos=() if "videos" in unsupported else ir_request.videos, - ) - return _UnsupportedParams(unsupported) + unsupported: Final = _unsupported_param_names(ir_request, provider_config, model, custom_llm_provider) + if unsupported: + return _UnsupportedParams(unsupported) + return ir_request def prepare_call( @@ -219,8 +228,10 @@ def prepare_call( except ValueError as error: raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error try: - request_safety_identifier: Final = _request_safety_identifier(safety_identifier, kwargs) - request: Final = fields.validate(request_safety_identifier) + request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs) + request: Final = _fields_without_unsupported_media( + fields, provider_config, canonical_model, provider, kwargs + ).validate(request_safety_identifier) except ValidationError as error: raise litellm.BadRequestError( message=f"Invalid Decisions request: {error}", @@ -248,7 +259,7 @@ def prepare_call( safety_identifier=request_safety_identifier, provider_config=provider_config, model=canonical_model, - kwargs=kwargs, + custom_llm_provider=provider, ) if isinstance(ir_request, _UnsupportedParams): raise litellm.UnsupportedParamsError( diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 33ca895c5e0..26d880c0d8a 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -8,6 +8,7 @@ import httpx from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never +import litellm from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( @@ -168,10 +169,12 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]: def ir_to_systemone_request( - model: str, request: DecisionsIRRequest, supports_audio_video: bool = False + model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - if (request.audio or request.videos) and not supports_audio_video: - return UnsupportedDecisionsRequest(reason="audio/video input is not supported by this Decisions model") + if request.audio and not supports_audio: + return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model") + if request.videos and not supports_video: + return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model") state: Final = _systemone_state(request.input) if isinstance(state, UnsupportedDecisionsRequest): return state @@ -369,8 +372,11 @@ class BaseDecisionsConfig(ABC): ) -> tuple[Mapping[str, str], bytes | None]: return headers, None - def supports_audio_video_input(self, model: str) -> bool: - return False + def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool: + return litellm.supports_audio_input(model=model, custom_llm_provider=custom_llm_provider) + + def supports_video_input(self, model: str, custom_llm_provider: str) -> bool: + return litellm.supports_video_input(model=model, custom_llm_provider=custom_llm_provider) def transform_decisions_request( self, @@ -378,7 +384,12 @@ class BaseDecisionsConfig(ABC): request: DecisionsIRRequest, custom_llm_provider: str, ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - return ir_to_systemone_request(self.request_model(model), request, self.supports_audio_video_input(model)) + return ir_to_systemone_request( + self.request_model(model), + request, + supports_audio=self.supports_audio_input(model, custom_llm_provider), + supports_video=self.supports_video_input(model, custom_llm_provider), + ) def unwrap_response(self, payload: object) -> object: return payload diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index 6e8a2ebd1ec..fa8c68ad3a4 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -30,9 +30,6 @@ class CloudflareDecisionsConfig(BaseDecisionsConfig): return model return f"@cf/cloudflare/{model}" - def supports_audio_video_input(self, model: str) -> bool: - return self.canonical_model(model) == "@cf/cloudflare/clef-omni" - def request_model(self, model: str) -> str: return model.rsplit("/", maxsplit=1)[-1] diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 842c2cc3e4c..9206b45ae48 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15988,7 +15988,9 @@ "max_input_tokens": 64000, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/" + "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/", + "supports_audio_input": true, + "supports_video_input": true }, "cloudflare/@cf/cloudflare/clef-omni": { "input_cost_per_token": 1.5e-07, @@ -15996,7 +15998,9 @@ "max_input_tokens": 64000, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/" + "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/", + "supports_audio_input": true, + "supports_video_input": true }, "cloudflare/clef": { "input_cost_per_token": 2.4e-07, diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index dbb7b69bd8b..6b6b7655719 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -92,8 +92,11 @@ async def _invalid_request( ) +_FIELDS_CHECKED_AFTER_ROUTING: Final = frozenset({"safety_identifier", "audio", "videos"}) + + def _fields_checked_before_routing(data: Mapping[str, object]) -> Mapping[str, object]: - return MappingProxyType({key: value for key, value in data.items() if key != "safety_identifier"}) + return MappingProxyType({key: value for key, value in data.items() if key not in _FIELDS_CHECKED_AFTER_ROUTING}) async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index fc6fe977fa3..bef61bee40d 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from types import MappingProxyType from typing import Annotated, Final, Literal, TypeAlias -from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config +from pydantic import ConfigDict, Field, PrivateAttr, field_validator, model_validator, with_config from typing_extensions import ReadOnly, Required, TypedDict from litellm.types.llms.base import LiteLLMPydanticObjectBase @@ -103,6 +103,11 @@ class DecisionsMediaBody(LiteLLMPydanticObjectBase): model_config = ConfigDict(extra="allow", frozen=True) + @field_validator("audio", "videos", mode="before") + @classmethod + def _null_means_no_clips(cls, value: object) -> object: + return () if value is None else value + class DecisionsRequestBody(DecisionsMediaBody): state: Annotated[ diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6f89ac614e8..60ac183ebb9 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -191,6 +191,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_thinking_cache_preservation: ReadOnly[bool | None] supports_computer_use: bool | None supports_audio_input: bool | None + supports_video_input: ReadOnly[bool | None] supports_embedding_image_input: bool | None supports_audio_output: bool | None supports_pdf_input: bool | None diff --git a/litellm/utils.py b/litellm/utils.py index 44dc75f45d7..a37747cde2e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2975,6 +2975,10 @@ def supports_audio_input(model: str, custom_llm_provider: str | None = None) -> return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_audio_input") +def supports_video_input(model: str, custom_llm_provider: str | None = None) -> bool: + return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_video_input") + + def supports_pdf_input(model: str, custom_llm_provider: str | None = None) -> bool: """Check if a given model supports pdf input in a chat completion call""" return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_pdf_input") @@ -6385,6 +6389,7 @@ def get_model_info_helper( supports_prompt_cache_breakpoint=_model_info.get("supports_prompt_cache_breakpoint", None), supports_thinking_cache_preservation=_model_info.get("supports_thinking_cache_preservation", None), supports_audio_input=_model_info.get("supports_audio_input", None), + supports_video_input=_model_info.get("supports_video_input", None), supports_audio_output=_model_info.get("supports_audio_output", None), supports_pdf_input=_model_info.get("supports_pdf_input", None), supports_embedding_image_input=_model_info.get("supports_embedding_image_input", None), @@ -6535,6 +6540,7 @@ def get_model_info( supports_prompt_caching: Optional[bool] supports_prompt_cache_breakpoint: Optional[bool] supports_audio_input: Optional[bool] + supports_video_input: Optional[bool] supports_audio_output: Optional[bool] supports_pdf_input: Optional[bool] supports_web_search: Optional[bool] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 842c2cc3e4c..9206b45ae48 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15988,7 +15988,9 @@ "max_input_tokens": 64000, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/" + "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/", + "supports_audio_input": true, + "supports_video_input": true }, "cloudflare/@cf/cloudflare/clef-omni": { "input_cost_per_token": 1.5e-07, @@ -15996,7 +15998,9 @@ "max_input_tokens": 64000, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/" + "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/", + "supports_audio_input": true, + "supports_video_input": true }, "cloudflare/clef": { "input_cost_per_token": 2.4e-07, diff --git a/tests/integration/providers/test_cloudflare_clef_omni_media_wire.py b/tests/integration/providers/test_cloudflare_clef_omni_media_wire.py index 4c46f908a73..b423b407422 100644 --- a/tests/integration/providers/test_cloudflare_clef_omni_media_wire.py +++ b/tests/integration/providers/test_cloudflare_clef_omni_media_wire.py @@ -303,11 +303,15 @@ def test_both_clef_omni_spellings_forward_media_and_carry_the_cost_map_price(gat info.get("output_cost_per_token"), info.get("mode"), info.get("max_input_tokens"), + info.get("supports_audio_input"), + info.get("supports_video_input"), ) == ( price["input_cost_per_token"], price["output_cost_per_token"], price["mode"], price["max_input_tokens"], + price["supports_audio_input"], + price["supports_video_input"], ), (model, info) @@ -323,20 +327,18 @@ def test_the_maxima_of_four_audio_and_two_video_clips_are_forwarded(gateway: Gat assert _upstream_body(request) == _expected_upstream_body(marker, audio=audio, videos=videos) -def test_missing_and_empty_media_send_the_same_text_only_body_and_null_is_refused(gateway: Gateway) -> None: +def test_missing_empty_and_null_media_send_the_same_text_only_body(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(_echo) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=_CANONICAL, api_base=wire.url, api_key=_API_KEY) missing: Final = _decide(gateway, "/v1/systemone", model, marker) empty: Final = _decide(gateway, "/v1/systemone", model, marker, audio=[], videos=[]) - for response in (missing, empty): + null: Final = _decide(gateway, "/v1/systemone", model, marker, audio=None, videos=None) + for response in (missing, empty, null): assert response.status_code == 200, response.text assert response.json() == _expected_response("/v1/systemone", marker) - null: Final = _decide(gateway, "/v1/systemone", model, marker, audio=None) - assert null.status_code == 400, null.text - assert "Invalid Decisions request" in null.text, null.text bodies: Final = tuple(_upstream_body(request) for request in wire.drain()) - assert bodies == (_expected_upstream_body(marker), _expected_upstream_body(marker)), bodies + assert bodies == (_expected_upstream_body(marker),) * 3, bodies _MALFORMED: Final[tuple[tuple[str, dict[str, JsonValue], str], ...]] = ( diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 1587c4424c5..9abe26e82b6 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -1780,20 +1780,29 @@ async def test_clips_for_a_model_without_media_support_are_refused_before_http( assert len(respx_mock.calls) == 0 +@pytest.mark.parametrize( + "clips", + ( + {"audio": ["data:audio/wav;base64,YQ=="], "videos": ["data:video/mp4;base64,YQ=="]}, + {"audio": ["https://example.com/clip.wav"], "videos": [{"content_type": "video/mp4"}]}, + ), + ids=("well_formed", "malformed"), +) @pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) @pytest.mark.asyncio async def test_clips_are_dropped_from_the_wire_for_a_model_without_media_support_under_drop_params( - api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + api: str, + request_kwargs: Mapping[str, object], + clips: Mapping[str, object], + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) call: Final = getattr(litellm, f"a{api}") - clips: Final[Mapping[str, object]] = MappingProxyType( - {"audio": ["data:audio/wav;base64,YQ=="], "videos": ["data:video/mp4;base64,YQ=="], "drop_params": True} - ) await call(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", **request_kwargs) - await call(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", **request_kwargs, **clips) + await call(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", drop_params=True, **request_kwargs, **clips) assert len(route.calls) == 2 text_only_body: Final = json.loads(route.calls[0].request.content) @@ -1830,8 +1839,13 @@ async def test_systemone_validates_media_before_http(respx_mock: respx.MockRoute assert len(respx_mock.calls) == 0 +@pytest.mark.parametrize( + "media", ({"audio": [], "videos": []}, {"audio": None, "videos": None}, {"audio": None, "videos": []}), ids=repr +) @pytest.mark.asyncio -async def test_empty_media_keeps_text_only_request_unchanged(respx_mock: respx.MockRouter) -> None: +async def test_empty_or_null_media_keeps_text_only_request_unchanged( + respx_mock: respx.MockRouter, media: Mapping[str, object] +) -> None: route: Final = respx_mock.post("https://example.com/v1/systemone").respond(200, json=dict(_RESPONSE)) await litellm.asystemone( model="typesafe/jev-latest", @@ -1839,11 +1853,68 @@ async def test_empty_media_keeps_text_only_request_unchanged(respx_mock: respx.M questions=_QUESTIONS, api_base="https://example.com", api_key="test", - audio=[], - videos=[], + **media, ) assert json.loads(route.calls[0].request.content) == { "model": "jev-latest", "state": "test", "questions": _QUESTIONS, } + + +@pytest.mark.asyncio +async def test_a_malformed_clip_for_a_model_that_takes_media_is_refused_even_under_drop_params( + respx_mock: respx.MockRouter, +) -> None: + with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"): + await litellm.asystemone( + model="cloudflare/clef-omni", + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + audio=["https://example.com/clip.wav"], + drop_params=True, + ) + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_media_support_is_read_per_kind_from_the_cost_map( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setitem( + litellm.model_cost, + "typesafe/jev-hears", + {"litellm_provider": "typesafe", "mode": "evaluation", "supports_audio_input": True}, + ) + route: Final = respx_mock.post("https://example.com/v1/systemone").respond(200, json=dict(_RESPONSE)) + audio: Final = ["data:audio/wav;base64,YQ=="] + + await litellm.asystemone( + model="typesafe/jev-hears", + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + audio=audio, + ) + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['videos']")): + await litellm.asystemone( + model="typesafe/jev-hears", + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + audio=audio, + videos=["data:video/mp4;base64,YQ=="], + ) + + assert len(route.calls) == 1 + assert json.loads(route.calls[0].request.content) == { + "model": "jev-hears", + "state": "test", + "questions": _QUESTIONS, + "audio": audio, + } diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index a452a621d2e..454fa393ea8 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -178,3 +178,23 @@ def test_missing_or_mismatched_systemone_answers_are_refusals_left_out_of_the_re assert parsed.answers[1:] == (DecisionsIRRefusal(), DecisionsIRRefusal(), DecisionsIRRefusal()) assert tuple(ir_to_systemone_response(parsed, ir).answers) == ("damaged",) + + +def test_each_media_kind_is_gated_by_its_own_support_flag() -> None: + audio: Final = ["data:audio/wav;base64,YQ=="] + videos: Final = ["data:video/mp4;base64,YQ=="] + audio_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "audio": audio})) + video_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "videos": videos})) + + assert ir_to_systemone_request("clef-omni", audio_only, supports_audio=True) == { + "model": "clef-omni", + **_SYSTEMONE_REQUEST, + "audio": audio, + } + assert ir_to_systemone_request("clef-omni", video_only, supports_video=True) == { + "model": "clef-omni", + **_SYSTEMONE_REQUEST, + "videos": videos, + } + assert isinstance(ir_to_systemone_request("clef-omni", audio_only, supports_video=True), UnsupportedDecisionsRequest) + assert isinstance(ir_to_systemone_request("clef-omni", video_only, supports_audio=True), UnsupportedDecisionsRequest) diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 21a565fd1b1..7c9cb53d845 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -794,6 +794,73 @@ def test_clips_for_a_model_without_media_support_are_refused_unless_the_deployme } +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone", "/v1/decisions", "/decisions")) +def test_null_media_on_every_decisions_route_means_no_clips( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + endpoint: str, +) -> None: + _route_to_clef_omni(monkeypatch) + upstream: Final = respx_mock.post(_CLEF_OMNI_UPSTREAM).respond(json=_CLEF_OMNI_RESPONSE) + + response: Final = client.post(endpoint, json={**_clef_omni_request(endpoint), "audio": None, "videos": None}) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "clef-omni", + "state": "Evaluate the clips", + "questions": {"heard": {"type": "noul", "instructions": "Is there speech?"}}, + } + + +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/v1/decisions")) +@pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params")) +def test_a_malformed_clip_for_a_model_without_media_support_is_dropped_only_when_the_deployment_drops_params( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + endpoint: str, + deployment_drops_params: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "clef-flash", + "litellm_params": { + "model": "cloudflare/clef-flash", + "api_base": "https://cf.example", + "api_key": "cf-key", + "drop_params": deployment_drops_params, + }, + } + ] + ), + ) + upstream: Final = respx_mock.post("https://cf.example/ai/run/@cf/cloudflare/clef-flash").respond( + json=_CLEF_OMNI_RESPONSE + ) + malformed: Final = {"audio": ["https://example.com/clip.wav"], "videos": [{"content_type": "video/mp4"}]} + + response: Final = client.post(endpoint, json={**_clef_omni_request(endpoint), "model": "clef-flash", **malformed}) + + if not deployment_drops_params: + assert response.status_code == 400, response.text + assert "Invalid Decisions request" in response.json()["error"]["message"] + assert not upstream.called + return + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "clef-flash", + "state": "Evaluate the clips", + "questions": {"heard": {"type": "noul", "instructions": "Is there speech?"}}, + } + + def _decisions_feature() -> LazyFeature: return next(feature for feature in LAZY_FEATURES if feature.name == "decisions") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7a192d17347..07bbeb872d4 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -437,6 +437,26 @@ def test_supports_audio_output_reads_its_own_cost_map_flag( assert litellm.supports_audio_output(model) is audio_output +@pytest.mark.parametrize(("audio_input", "video_input"), [(True, False), (False, True)]) +def test_supports_video_input_reads_its_own_cost_map_flag( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, audio_input: bool, video_input: bool +) -> None: + model: Final = "openai/video-flags-disagree-model" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "openai", + "mode": "chat", + "supports_audio_input": audio_input, + "supports_video_input": video_input, + }, + ) + assert litellm.supports_video_input(model) is video_input + assert litellm.supports_audio_input(model) is audio_input + assert litellm.get_model_info(model)["supports_video_input"] is video_input + + def test_get_optional_params_image_gen(): from litellm.llms.azure.image_generation import AzureGPTImageGenerationConfig