From bba75aa12b98598fe357e52cb8f5df1de13bfd94 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Wed, 2 Jul 2025 20:37:45 -0700 Subject: [PATCH] Add 'audio_url' message type support for VLLM (#12270) * fix(openai.py): add audio_url content type for vllm Fixes https://github.com/BerriAI/litellm/issues/12196 * test: fix test --- litellm/proxy/_new_secret_config.yaml | 5 ++ litellm/types/llms/openai.py | 1 + tests/llm_translation/test_bedrock_agents.py | 50 ++++++++++++++----- .../test_hosted_vllm_chat_transformation.py | 43 ++++++++++++++++ 4 files changed, 87 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 9630cf3eef1..ca129503bc0 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -3,3 +3,8 @@ model_list: litellm_params: model: "gpt-4o" api_key: os.environ/OPENAI_API_KEY + - model_name: "gemini-2.0-flash" + litellm_params: + model: "gemini/gemini-2.0-flash" + api_key: os.environ/GEMINI_API_KEY + stream: True diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 775583ddc40..97c940c71ac 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -705,6 +705,7 @@ ValidUserMessageContentTypes = [ "text", "image_url", "input_audio", + "audio_url", "document", "video_url", "file", diff --git a/tests/llm_translation/test_bedrock_agents.py b/tests/llm_translation/test_bedrock_agents.py index 5b0de570c36..46390e804d7 100644 --- a/tests/llm_translation/test_bedrock_agents.py +++ b/tests/llm_translation/test_bedrock_agents.py @@ -18,18 +18,14 @@ from unittest.mock import AsyncMock, Mock, patch import pytest + @pytest.mark.asyncio async def test_bedrock_agents(): litellm._turn_on_debug() response = litellm.completion( - model="bedrock/agent/L1RT58GYRW/MFPSBCXYTW", - messages=[ - { - "role": "user", - "content": "Hi just respond with a ping message" - } - ], - ) + model="bedrock/agent/L1RT58GYRW/MFPSBCXYTW", + messages=[{"role": "user", "content": "Hi just respond with a ping message"}], + ) ######################################################### ######################################################### @@ -38,12 +34,15 @@ async def test_bedrock_agents(): # assert that the message content has a response with some length assert len(response.choices[0].message.content) > 0 - # assert we were able to get the response cost - assert response._hidden_params["response_cost"] is not None and response._hidden_params["response_cost"] > 0 + assert ( + response._hidden_params["response_cost"] is not None + and response._hidden_params["response_cost"] > 0 + ) pass + @pytest.mark.asyncio async def test_bedrock_agents_with_streaming(): # litellm._turn_on_debug() @@ -52,7 +51,7 @@ async def test_bedrock_agents_with_streaming(): messages=[ { "role": "user", - "content": "Hi who is ishaan cto of litellm, tell me 10 things about him" + "content": "Hi who is ishaan cto of litellm, tell me 10 things about him", } ], stream=True, @@ -61,4 +60,31 @@ async def test_bedrock_agents_with_streaming(): for chunk in response: print("final chunk=", chunk) - pass \ No newline at end of file + pass + + +def test_bedrock_agents_with_custom_params(): + litellm._turn_on_debug() + from unittest.mock import MagicMock, patch + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + + with patch.object(client, "post", return_value=MagicMock()) as mock_post: + try: + response = litellm.completion( + model="bedrock/agent/L1RT58GYRW/MFPSBCXYTW", + messages=[ + { + "role": "user", + "content": "Hi who is ishaan cto of litellm, tell me 10 things about him", + } + ], + invocationId="my-test-invocation-id", + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_post.assert_called_once() + print(f"mock_post.call_args.kwargs: {mock_post.call_args.kwargs}") diff --git a/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py b/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py index 7885195b5b8..01acd144305 100644 --- a/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py +++ b/tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py @@ -43,3 +43,46 @@ def test_hosted_vllm_chat_transformation_file_url(): "content": [{"type": "video_url", "video_url": {"url": video_data}}], } ] + + +def test_hosted_vllm_chat_transformation_with_audio_url(): + from litellm import completion + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = MagicMock() + + with patch.object( + client.chat.completions.with_raw_response, "create", return_value=MagicMock() + ) as mock_post: + try: + response = completion( + model="hosted_vllm/llama-3.1-70b-instruct", + messages=[ + { + "role": "user", + "content": [ + { + "type": "audio_url", + "audio_url": {"url": "https://example.com/audio.mp3"}, + }, + ], + }, + ], + client=client, + ) + except Exception as e: + print(f"Error: {e}") + + mock_post.assert_called_once() + print(f"mock_post.call_args.kwargs: {mock_post.call_args.kwargs}") + assert mock_post.call_args.kwargs["messages"] == [ + { + "role": "user", + "content": [ + { + "type": "audio_url", + "audio_url": {"url": "https://example.com/audio.mp3"}, + } + ], + } + ]