diff --git a/litellm/__init__.py b/litellm/__init__.py index a611b90bc37..ceae8388eee 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -2276,6 +2276,7 @@ if TYPE_CHECKING: supports_parallel_function_calling: Callable[..., bool] supports_vision: Callable[..., bool] supports_audio_input: Callable[..., bool] + supports_video_input: Callable[..., bool] supports_audio_output: Callable[..., bool] supports_system_messages: Callable[..., bool] supports_reasoning: Callable[..., bool] diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index ce69de3b240..0e7cdd115b3 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -36,6 +36,7 @@ UTILS_NAMES: Final = ( "supports_parallel_function_calling", "supports_vision", "supports_audio_input", + "supports_video_input", "supports_audio_output", "supports_system_messages", "supports_reasoning", @@ -481,6 +482,7 @@ _UTILS_IMPORT_MAP: Final = { ), "supports_vision": (".utils", "supports_vision"), "supports_audio_input": (".utils", "supports_audio_input"), + "supports_video_input": (".utils", "supports_video_input"), "supports_audio_output": (".utils", "supports_audio_output"), "supports_system_messages": (".utils", "supports_system_messages"), "supports_reasoning": (".utils", "supports_reasoning"), diff --git a/litellm/decisions/call.py b/litellm/decisions/call.py index a5c265102fd..6d014bebee3 100644 --- a/litellm/decisions/call.py +++ b/litellm/decisions/call.py @@ -49,10 +49,6 @@ _SAFETY_IDENTIFIER_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter( _HANDLER: Final = BaseLLMHTTPHandler() -def _media_fields(audio: DecisionsAudioClips | None, videos: DecisionsVideoClips | None) -> Mapping[str, object]: - return {"audio": audio if audio is not None else (), "videos": videos if videos is not None else ()} - - @dataclass(frozen=True, slots=True) class SystemOneRequestFields: state: DecisionsJSON | None @@ -62,7 +58,7 @@ class SystemOneRequestFields: def validate(self, safety_identifier: str | None) -> DecisionsRequestBody: return _SYSTEMONE_REQUEST_ADAPTER.validate_python( - {"state": self.state, "questions": self.questions, **_media_fields(self.audio, self.videos)} + {"state": self.state, "questions": self.questions, "audio": self.audio, "videos": self.videos} ) @@ -79,7 +75,8 @@ class OpenAIDecisionRequestFields: "input": self.input, "questions": self.questions, "safety_identifier": safety_identifier, - **_media_fields(self.audio, self.videos), + "audio": self.audio, + "videos": self.videos, } ) @@ -141,28 +138,47 @@ def _drops_params(kwargs: Mapping[str, object]) -> bool: return litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True -def _request_safety_identifier(safety_identifier: object, kwargs: Mapping[str, object]) -> str | None: - if isinstance(safety_identifier, str) or not _drops_params(kwargs): +def _request_safety_identifier( + safety_identifier: object, provider_config: BaseDecisionsConfig, kwargs: Mapping[str, object] +) -> str | None: + if not _drops_params(kwargs): return _SAFETY_IDENTIFIER_ADAPTER.validate_python(safety_identifier) + if isinstance(safety_identifier, str) and provider_config.supports_safety_identifier: + return safety_identifier return None +def _fields_without_unsupported_media( + fields: RequestFields, + provider_config: BaseDecisionsConfig, + model: str, + custom_llm_provider: str, + kwargs: Mapping[str, object], +) -> RequestFields: + if not _drops_params(kwargs): + return fields + return replace( + fields, + audio=fields.audio if provider_config.supports_audio_input(model, custom_llm_provider) else None, + videos=fields.videos if provider_config.supports_video_input(model, custom_llm_provider) else None, + ) + + @dataclass(frozen=True, slots=True) class _UnsupportedParams: names: tuple[str, ...] def _unsupported_param_names( - ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, model: str + ir_request: DecisionsIRRequest, provider_config: BaseDecisionsConfig, model: str, custom_llm_provider: str ) -> tuple[str, ...]: - takes_media: Final = provider_config.supports_audio_video_input(model) checks: Final = ( ( "safety_identifier", ir_request.safety_identifier is not None and not provider_config.supports_safety_identifier, ), - ("audio", bool(ir_request.audio) and not takes_media), - ("videos", bool(ir_request.videos) and not takes_media), + ("audio", bool(ir_request.audio) and not provider_config.supports_audio_input(model, custom_llm_provider)), + ("videos", bool(ir_request.videos) and not provider_config.supports_video_input(model, custom_llm_provider)), ) return tuple(name for name, unsupported in checks if unsupported) @@ -173,20 +189,13 @@ def _provider_ir_request( safety_identifier: str | None, provider_config: BaseDecisionsConfig, model: str, - kwargs: Mapping[str, object], + custom_llm_provider: str, ) -> DecisionsIRRequest | _UnsupportedParams: ir_request: Final = _ir_request(request, safety_identifier) - unsupported: Final = _unsupported_param_names(ir_request, provider_config, model) - if not unsupported: - return ir_request - if _drops_params(kwargs): - return replace( - ir_request, - safety_identifier=None if "safety_identifier" in unsupported else ir_request.safety_identifier, - audio=() if "audio" in unsupported else ir_request.audio, - videos=() if "videos" in unsupported else ir_request.videos, - ) - return _UnsupportedParams(unsupported) + unsupported: Final = _unsupported_param_names(ir_request, provider_config, model, custom_llm_provider) + if unsupported: + return _UnsupportedParams(unsupported) + return ir_request def prepare_call( @@ -219,8 +228,10 @@ def prepare_call( except ValueError as error: raise litellm.BadRequestError(message=str(error), model=model, llm_provider=provider) from error try: - request_safety_identifier: Final = _request_safety_identifier(safety_identifier, kwargs) - request: Final = fields.validate(request_safety_identifier) + request_safety_identifier: Final = _request_safety_identifier(safety_identifier, provider_config, kwargs) + request: Final = _fields_without_unsupported_media( + fields, provider_config, canonical_model, provider, kwargs + ).validate(request_safety_identifier) except ValidationError as error: raise litellm.BadRequestError( message=f"Invalid Decisions request: {error}", @@ -248,7 +259,7 @@ def prepare_call( safety_identifier=request_safety_identifier, provider_config=provider_config, model=canonical_model, - kwargs=kwargs, + custom_llm_provider=provider, ) if isinstance(ir_request, _UnsupportedParams): raise litellm.UnsupportedParamsError( diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 33ca895c5e0..26d880c0d8a 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -8,6 +8,7 @@ import httpx from pydantic import TypeAdapter, ValidationError from typing_extensions import assert_never +import litellm from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str from litellm.types.decisions import ( @@ -168,10 +169,12 @@ def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]: def ir_to_systemone_request( - model: str, request: DecisionsIRRequest, supports_audio_video: bool = False + model: str, request: DecisionsIRRequest, *, supports_audio: bool = False, supports_video: bool = False ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - if (request.audio or request.videos) and not supports_audio_video: - return UnsupportedDecisionsRequest(reason="audio/video input is not supported by this Decisions model") + if request.audio and not supports_audio: + return UnsupportedDecisionsRequest(reason="audio input is not supported by this Decisions model") + if request.videos and not supports_video: + return UnsupportedDecisionsRequest(reason="video input is not supported by this Decisions model") state: Final = _systemone_state(request.input) if isinstance(state, UnsupportedDecisionsRequest): return state @@ -369,8 +372,11 @@ class BaseDecisionsConfig(ABC): ) -> tuple[Mapping[str, str], bytes | None]: return headers, None - def supports_audio_video_input(self, model: str) -> bool: - return False + def supports_audio_input(self, model: str, custom_llm_provider: str) -> bool: + return litellm.supports_audio_input(model=model, custom_llm_provider=custom_llm_provider) + + def supports_video_input(self, model: str, custom_llm_provider: str) -> bool: + return litellm.supports_video_input(model=model, custom_llm_provider=custom_llm_provider) def transform_decisions_request( self, @@ -378,7 +384,12 @@ class BaseDecisionsConfig(ABC): request: DecisionsIRRequest, custom_llm_provider: str, ) -> Mapping[str, object] | UnsupportedDecisionsRequest: - return ir_to_systemone_request(self.request_model(model), request, self.supports_audio_video_input(model)) + return ir_to_systemone_request( + self.request_model(model), + request, + supports_audio=self.supports_audio_input(model, custom_llm_provider), + supports_video=self.supports_video_input(model, custom_llm_provider), + ) def unwrap_response(self, payload: object) -> object: return payload diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index 6e8a2ebd1ec..fa8c68ad3a4 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -30,9 +30,6 @@ class CloudflareDecisionsConfig(BaseDecisionsConfig): return model return f"@cf/cloudflare/{model}" - def supports_audio_video_input(self, model: str) -> bool: - return self.canonical_model(model) == "@cf/cloudflare/clef-omni" - def request_model(self, model: str) -> str: return model.rsplit("/", maxsplit=1)[-1] diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 842c2cc3e4c..9206b45ae48 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -15988,7 +15988,9 @@ "max_input_tokens": 64000, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/" + "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/", + "supports_audio_input": true, + "supports_video_input": true }, "cloudflare/@cf/cloudflare/clef-omni": { "input_cost_per_token": 1.5e-07, @@ -15996,7 +15998,9 @@ "max_input_tokens": 64000, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/" + "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/", + "supports_audio_input": true, + "supports_video_input": true }, "cloudflare/clef": { "input_cost_per_token": 2.4e-07, diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index dbb7b69bd8b..6b6b7655719 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -92,8 +92,11 @@ async def _invalid_request( ) +_FIELDS_CHECKED_AFTER_ROUTING: Final = frozenset({"safety_identifier", "audio", "videos"}) + + def _fields_checked_before_routing(data: Mapping[str, object]) -> Mapping[str, object]: - return MappingProxyType({key: value for key, value in data.items() if key != "safety_identifier"}) + return MappingProxyType({key: value for key, value in data.items() if key not in _FIELDS_CHECKED_AFTER_ROUTING}) async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index fc6fe977fa3..bef61bee40d 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -3,7 +3,7 @@ from dataclasses import dataclass, field from types import MappingProxyType from typing import Annotated, Final, Literal, TypeAlias -from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config +from pydantic import ConfigDict, Field, PrivateAttr, field_validator, model_validator, with_config from typing_extensions import ReadOnly, Required, TypedDict from litellm.types.llms.base import LiteLLMPydanticObjectBase @@ -103,6 +103,11 @@ class DecisionsMediaBody(LiteLLMPydanticObjectBase): model_config = ConfigDict(extra="allow", frozen=True) + @field_validator("audio", "videos", mode="before") + @classmethod + def _null_means_no_clips(cls, value: object) -> object: + return () if value is None else value + class DecisionsRequestBody(DecisionsMediaBody): state: Annotated[ diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 6f89ac614e8..60ac183ebb9 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -191,6 +191,7 @@ class ProviderSpecificModelInfo(TypedDict, total=False): supports_thinking_cache_preservation: ReadOnly[bool | None] supports_computer_use: bool | None supports_audio_input: bool | None + supports_video_input: ReadOnly[bool | None] supports_embedding_image_input: bool | None supports_audio_output: bool | None supports_pdf_input: bool | None diff --git a/litellm/utils.py b/litellm/utils.py index 44dc75f45d7..a37747cde2e 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2975,6 +2975,10 @@ def supports_audio_input(model: str, custom_llm_provider: str | None = None) -> return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_audio_input") +def supports_video_input(model: str, custom_llm_provider: str | None = None) -> bool: + return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_video_input") + + def supports_pdf_input(model: str, custom_llm_provider: str | None = None) -> bool: """Check if a given model supports pdf input in a chat completion call""" return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_pdf_input") @@ -6385,6 +6389,7 @@ def get_model_info_helper( supports_prompt_cache_breakpoint=_model_info.get("supports_prompt_cache_breakpoint", None), supports_thinking_cache_preservation=_model_info.get("supports_thinking_cache_preservation", None), supports_audio_input=_model_info.get("supports_audio_input", None), + supports_video_input=_model_info.get("supports_video_input", None), supports_audio_output=_model_info.get("supports_audio_output", None), supports_pdf_input=_model_info.get("supports_pdf_input", None), supports_embedding_image_input=_model_info.get("supports_embedding_image_input", None), @@ -6535,6 +6540,7 @@ def get_model_info( supports_prompt_caching: Optional[bool] supports_prompt_cache_breakpoint: Optional[bool] supports_audio_input: Optional[bool] + supports_video_input: Optional[bool] supports_audio_output: Optional[bool] supports_pdf_input: Optional[bool] supports_web_search: Optional[bool] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 842c2cc3e4c..9206b45ae48 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -15988,7 +15988,9 @@ "max_input_tokens": 64000, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/" + "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/", + "supports_audio_input": true, + "supports_video_input": true }, "cloudflare/@cf/cloudflare/clef-omni": { "input_cost_per_token": 1.5e-07, @@ -15996,7 +15998,9 @@ "max_input_tokens": 64000, "mode": "evaluation", "output_cost_per_token": 0.0, - "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/" + "source": "https://developers.cloudflare.com/workers-ai/models/clef-omni/", + "supports_audio_input": true, + "supports_video_input": true }, "cloudflare/clef": { "input_cost_per_token": 2.4e-07, diff --git a/tests/integration/providers/test_cloudflare_clef_omni_media_wire.py b/tests/integration/providers/test_cloudflare_clef_omni_media_wire.py index 4c46f908a73..b423b407422 100644 --- a/tests/integration/providers/test_cloudflare_clef_omni_media_wire.py +++ b/tests/integration/providers/test_cloudflare_clef_omni_media_wire.py @@ -303,11 +303,15 @@ def test_both_clef_omni_spellings_forward_media_and_carry_the_cost_map_price(gat info.get("output_cost_per_token"), info.get("mode"), info.get("max_input_tokens"), + info.get("supports_audio_input"), + info.get("supports_video_input"), ) == ( price["input_cost_per_token"], price["output_cost_per_token"], price["mode"], price["max_input_tokens"], + price["supports_audio_input"], + price["supports_video_input"], ), (model, info) @@ -323,20 +327,18 @@ def test_the_maxima_of_four_audio_and_two_video_clips_are_forwarded(gateway: Gat assert _upstream_body(request) == _expected_upstream_body(marker, audio=audio, videos=videos) -def test_missing_and_empty_media_send_the_same_text_only_body_and_null_is_refused(gateway: Gateway) -> None: +def test_missing_empty_and_null_media_send_the_same_text_only_body(gateway: Gateway) -> None: marker: Final = uuid.uuid4().hex with wire_server(_echo) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=_CANONICAL, api_base=wire.url, api_key=_API_KEY) missing: Final = _decide(gateway, "/v1/systemone", model, marker) empty: Final = _decide(gateway, "/v1/systemone", model, marker, audio=[], videos=[]) - for response in (missing, empty): + null: Final = _decide(gateway, "/v1/systemone", model, marker, audio=None, videos=None) + for response in (missing, empty, null): assert response.status_code == 200, response.text assert response.json() == _expected_response("/v1/systemone", marker) - null: Final = _decide(gateway, "/v1/systemone", model, marker, audio=None) - assert null.status_code == 400, null.text - assert "Invalid Decisions request" in null.text, null.text bodies: Final = tuple(_upstream_body(request) for request in wire.drain()) - assert bodies == (_expected_upstream_body(marker), _expected_upstream_body(marker)), bodies + assert bodies == (_expected_upstream_body(marker),) * 3, bodies _MALFORMED: Final[tuple[tuple[str, dict[str, JsonValue], str], ...]] = ( diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 1587c4424c5..9abe26e82b6 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -1780,20 +1780,29 @@ async def test_clips_for_a_model_without_media_support_are_refused_before_http( assert len(respx_mock.calls) == 0 +@pytest.mark.parametrize( + "clips", + ( + {"audio": ["data:audio/wav;base64,YQ=="], "videos": ["data:video/mp4;base64,YQ=="]}, + {"audio": ["https://example.com/clip.wav"], "videos": [{"content_type": "video/mp4"}]}, + ), + ids=("well_formed", "malformed"), +) @pytest.mark.parametrize(("api", "request_kwargs"), _PREDICATE_REQUESTS, ids=_PREDICATE_REQUEST_IDS) @pytest.mark.asyncio async def test_clips_are_dropped_from_the_wire_for_a_model_without_media_support_under_drop_params( - api: str, request_kwargs: Mapping[str, object], respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch + api: str, + request_kwargs: Mapping[str, object], + clips: Mapping[str, object], + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr(litellm, "drop_params", False) route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) call: Final = getattr(litellm, f"a{api}") - clips: Final[Mapping[str, object]] = MappingProxyType( - {"audio": ["data:audio/wav;base64,YQ=="], "videos": ["data:video/mp4;base64,YQ=="], "drop_params": True} - ) await call(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", **request_kwargs) - await call(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", **request_kwargs, **clips) + await call(model="perplexity/pplx-decider-v1-27b", api_key="caller-key", drop_params=True, **request_kwargs, **clips) assert len(route.calls) == 2 text_only_body: Final = json.loads(route.calls[0].request.content) @@ -1830,8 +1839,13 @@ async def test_systemone_validates_media_before_http(respx_mock: respx.MockRoute assert len(respx_mock.calls) == 0 +@pytest.mark.parametrize( + "media", ({"audio": [], "videos": []}, {"audio": None, "videos": None}, {"audio": None, "videos": []}), ids=repr +) @pytest.mark.asyncio -async def test_empty_media_keeps_text_only_request_unchanged(respx_mock: respx.MockRouter) -> None: +async def test_empty_or_null_media_keeps_text_only_request_unchanged( + respx_mock: respx.MockRouter, media: Mapping[str, object] +) -> None: route: Final = respx_mock.post("https://example.com/v1/systemone").respond(200, json=dict(_RESPONSE)) await litellm.asystemone( model="typesafe/jev-latest", @@ -1839,11 +1853,68 @@ async def test_empty_media_keeps_text_only_request_unchanged(respx_mock: respx.M questions=_QUESTIONS, api_base="https://example.com", api_key="test", - audio=[], - videos=[], + **media, ) assert json.loads(route.calls[0].request.content) == { "model": "jev-latest", "state": "test", "questions": _QUESTIONS, } + + +@pytest.mark.asyncio +async def test_a_malformed_clip_for_a_model_that_takes_media_is_refused_even_under_drop_params( + respx_mock: respx.MockRouter, +) -> None: + with pytest.raises(litellm.BadRequestError, match="Invalid Decisions request"): + await litellm.asystemone( + model="cloudflare/clef-omni", + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + audio=["https://example.com/clip.wav"], + drop_params=True, + ) + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_media_support_is_read_per_kind_from_the_cost_map( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, local_model_cost_map: None +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setitem( + litellm.model_cost, + "typesafe/jev-hears", + {"litellm_provider": "typesafe", "mode": "evaluation", "supports_audio_input": True}, + ) + route: Final = respx_mock.post("https://example.com/v1/systemone").respond(200, json=dict(_RESPONSE)) + audio: Final = ["data:audio/wav;base64,YQ=="] + + await litellm.asystemone( + model="typesafe/jev-hears", + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + audio=audio, + ) + with pytest.raises(litellm.UnsupportedParamsError, match=re.escape("does not support parameters: ['videos']")): + await litellm.asystemone( + model="typesafe/jev-hears", + state="test", + questions=_QUESTIONS, + api_base="https://example.com", + api_key="test", + audio=audio, + videos=["data:video/mp4;base64,YQ=="], + ) + + assert len(route.calls) == 1 + assert json.loads(route.calls[0].request.content) == { + "model": "jev-hears", + "state": "test", + "questions": _QUESTIONS, + "audio": audio, + } diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py index a452a621d2e..454fa393ea8 100644 --- a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -178,3 +178,23 @@ def test_missing_or_mismatched_systemone_answers_are_refusals_left_out_of_the_re assert parsed.answers[1:] == (DecisionsIRRefusal(), DecisionsIRRefusal(), DecisionsIRRefusal()) assert tuple(ir_to_systemone_response(parsed, ir).answers) == ("damaged",) + + +def test_each_media_kind_is_gated_by_its_own_support_flag() -> None: + audio: Final = ["data:audio/wav;base64,YQ=="] + videos: Final = ["data:video/mp4;base64,YQ=="] + audio_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "audio": audio})) + video_only: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python({**_SYSTEMONE_REQUEST, "videos": videos})) + + assert ir_to_systemone_request("clef-omni", audio_only, supports_audio=True) == { + "model": "clef-omni", + **_SYSTEMONE_REQUEST, + "audio": audio, + } + assert ir_to_systemone_request("clef-omni", video_only, supports_video=True) == { + "model": "clef-omni", + **_SYSTEMONE_REQUEST, + "videos": videos, + } + assert isinstance(ir_to_systemone_request("clef-omni", audio_only, supports_video=True), UnsupportedDecisionsRequest) + assert isinstance(ir_to_systemone_request("clef-omni", video_only, supports_audio=True), UnsupportedDecisionsRequest) diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 21a565fd1b1..7c9cb53d845 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -794,6 +794,73 @@ def test_clips_for_a_model_without_media_support_are_refused_unless_the_deployme } +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone", "/v1/decisions", "/decisions")) +def test_null_media_on_every_decisions_route_means_no_clips( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + endpoint: str, +) -> None: + _route_to_clef_omni(monkeypatch) + upstream: Final = respx_mock.post(_CLEF_OMNI_UPSTREAM).respond(json=_CLEF_OMNI_RESPONSE) + + response: Final = client.post(endpoint, json={**_clef_omni_request(endpoint), "audio": None, "videos": None}) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "clef-omni", + "state": "Evaluate the clips", + "questions": {"heard": {"type": "noul", "instructions": "Is there speech?"}}, + } + + +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/v1/decisions")) +@pytest.mark.parametrize("deployment_drops_params", (False, True), ids=("strict", "drop_params")) +def test_a_malformed_clip_for_a_model_without_media_support_is_dropped_only_when_the_deployment_drops_params( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + endpoint: str, + deployment_drops_params: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[ + { + "model_name": "clef-flash", + "litellm_params": { + "model": "cloudflare/clef-flash", + "api_base": "https://cf.example", + "api_key": "cf-key", + "drop_params": deployment_drops_params, + }, + } + ] + ), + ) + upstream: Final = respx_mock.post("https://cf.example/ai/run/@cf/cloudflare/clef-flash").respond( + json=_CLEF_OMNI_RESPONSE + ) + malformed: Final = {"audio": ["https://example.com/clip.wav"], "videos": [{"content_type": "video/mp4"}]} + + response: Final = client.post(endpoint, json={**_clef_omni_request(endpoint), "model": "clef-flash", **malformed}) + + if not deployment_drops_params: + assert response.status_code == 400, response.text + assert "Invalid Decisions request" in response.json()["error"]["message"] + assert not upstream.called + return + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "clef-flash", + "state": "Evaluate the clips", + "questions": {"heard": {"type": "noul", "instructions": "Is there speech?"}}, + } + + def _decisions_feature() -> LazyFeature: return next(feature for feature in LAZY_FEATURES if feature.name == "decisions") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7a192d17347..07bbeb872d4 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -437,6 +437,26 @@ def test_supports_audio_output_reads_its_own_cost_map_flag( assert litellm.supports_audio_output(model) is audio_output +@pytest.mark.parametrize(("audio_input", "video_input"), [(True, False), (False, True)]) +def test_supports_video_input_reads_its_own_cost_map_flag( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, audio_input: bool, video_input: bool +) -> None: + model: Final = "openai/video-flags-disagree-model" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "openai", + "mode": "chat", + "supports_audio_input": audio_input, + "supports_video_input": video_input, + }, + ) + assert litellm.supports_video_input(model) is video_input + assert litellm.supports_audio_input(model) is audio_input + assert litellm.get_model_info(model)["supports_video_input"] is video_input + + def test_get_optional_params_image_gen(): from litellm.llms.azure.image_generation import AzureGPTImageGenerationConfig