Add 'audio_url' message type support for VLLM (#12270)

* fix(openai.py): add audio_url content type for vllm

Fixes https://github.com/BerriAI/litellm/issues/12196

* test: fix test

litellm/proxy/_new_secret_config.yaml

@@ -3,3 +3,8 @@ model_list:
     litellm_params:
       model: "gpt-4o"
       api_key: os.environ/OPENAI_API_KEY
+  - model_name: "gemini-2.0-flash"
+    litellm_params:
+      model: "gemini/gemini-2.0-flash"
+      api_key: os.environ/GEMINI_API_KEY
+      stream: True

litellm/types/llms/openai.py

@@ -705,6 +705,7 @@ ValidUserMessageContentTypes = [
     "text",
     "image_url",
     "input_audio",
+    "audio_url",
     "document",
     "video_url",
     "file",

tests/llm_translation/test_bedrock_agents.py

@@ -18,18 +18,14 @@ from unittest.mock import AsyncMock, Mock, patch
 
 import pytest
 
+
 @pytest.mark.asyncio
 async def test_bedrock_agents():
     litellm._turn_on_debug()
     response = litellm.completion(
-            model="bedrock/agent/L1RT58GYRW/MFPSBCXYTW",
-            messages=[
-                {
-                    "role": "user",
-                    "content": "Hi just respond with a ping message"
-                }
-            ],
-        )
+        model="bedrock/agent/L1RT58GYRW/MFPSBCXYTW",
+        messages=[{"role": "user", "content": "Hi just respond with a ping message"}],
+    )
 
     #########################################################
     #########################################################
@@ -38,12 +34,15 @@ async def test_bedrock_agents():
     # assert that the message content has a response with some length
     assert len(response.choices[0].message.content) > 0
 
-
     # assert we were able to get the response cost
-    assert response._hidden_params["response_cost"] is not None and response._hidden_params["response_cost"] > 0
+    assert (
+        response._hidden_params["response_cost"] is not None
+        and response._hidden_params["response_cost"] > 0
+    )
 
     pass
 
+
 @pytest.mark.asyncio
 async def test_bedrock_agents_with_streaming():
     # litellm._turn_on_debug()
@@ -52,7 +51,7 @@ async def test_bedrock_agents_with_streaming():
         messages=[
             {
                 "role": "user",
-                "content": "Hi who is ishaan cto of litellm, tell me 10 things about him"
+                "content": "Hi who is ishaan cto of litellm, tell me 10 things about him",
             }
         ],
         stream=True,
@@ -61,4 +60,31 @@ async def test_bedrock_agents_with_streaming():
     for chunk in response:
         print("final chunk=", chunk)
 
-    pass
+    pass
+
+
+def test_bedrock_agents_with_custom_params():
+    litellm._turn_on_debug()
+    from unittest.mock import MagicMock, patch
+    from litellm.llms.custom_httpx.http_handler import HTTPHandler
+
+    client = HTTPHandler()
+
+    with patch.object(client, "post", return_value=MagicMock()) as mock_post:
+        try:
+            response = litellm.completion(
+                model="bedrock/agent/L1RT58GYRW/MFPSBCXYTW",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": "Hi who is ishaan cto of litellm, tell me 10 things about him",
+                    }
+                ],
+                invocationId="my-test-invocation-id",
+                client=client,
+            )
+        except Exception as e:
+            print(f"Error: {e}")
+
+        mock_post.assert_called_once()
+        print(f"mock_post.call_args.kwargs: {mock_post.call_args.kwargs}")

tests/test_litellm/llms/hosted_vllm/chat/test_hosted_vllm_chat_transformation.py

@@ -43,3 +43,46 @@ def test_hosted_vllm_chat_transformation_file_url():
             "content": [{"type": "video_url", "video_url": {"url": video_data}}],
         }
     ]
+
+
+def test_hosted_vllm_chat_transformation_with_audio_url():
+    from litellm import completion
+    from litellm.llms.custom_httpx.http_handler import HTTPHandler
+
+    client = MagicMock()
+
+    with patch.object(
+        client.chat.completions.with_raw_response, "create", return_value=MagicMock()
+    ) as mock_post:
+        try:
+            response = completion(
+                model="hosted_vllm/llama-3.1-70b-instruct",
+                messages=[
+                    {
+                        "role": "user",
+                        "content": [
+                            {
+                                "type": "audio_url",
+                                "audio_url": {"url": "https://example.com/audio.mp3"},
+                            },
+                        ],
+                    },
+                ],
+                client=client,
+            )
+        except Exception as e:
+            print(f"Error: {e}")
+
+        mock_post.assert_called_once()
+        print(f"mock_post.call_args.kwargs: {mock_post.call_args.kwargs}")
+        assert mock_post.call_args.kwargs["messages"] == [
+            {
+                "role": "user",
+                "content": [
+                    {
+                        "type": "audio_url",
+                        "audio_url": {"url": "https://example.com/audio.mp3"},
+                    }
+                ],
+            }
+        ]