diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index c506192f253..e14da8af3f1 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -49,6 +49,7 @@ from litellm.types.llms.openai import ( AllMessageValues, ChatCompletionAnnotation, ChatCompletionAnnotationURLCitation, + ChatCompletionAudioDelta, ChatCompletionResponseMessage, ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, @@ -2568,6 +2569,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): functions: ChatCompletionToolCallFunctionChunk | None, chat_completion_logprobs: ChoiceLogprobs | None, image_response: list[ImageURLListItem] | None, + audio_response: ChatCompletionAudioResponse | None = None, ) -> StreamingChoices: """ Helper method to create a streaming choice object for Vertex AI @@ -2586,6 +2588,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): content=chat_completion_message.get("content"), reasoning_content=chat_completion_message.get("reasoning_content"), tool_calls=tools, + audio=ChatCompletionAudioDelta(data=audio_response.data) if audio_response is not None else None, images=image_response, function_call=functions, annotations=annotations, @@ -2795,6 +2798,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): continue image_response: list[ImageURLListItem] | None = None + audio_response: ChatCompletionAudioResponse | None = None chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"} chat_completion_logprobs: ChoiceLogprobs | None = None tools: list[ChatCompletionToolCallChunk] | None = None @@ -2921,6 +2925,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): functions=functions, chat_completion_logprobs=chat_completion_logprobs, image_response=image_response, + audio_response=audio_response, ) model_response.choices.append(choice) elif isinstance(model_response, ModelResponse): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5905aaa3380..83a3a9f1081 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -86,6 +86,7 @@ from .llms.openai import ( AllMessageValues, Batch, ChatCompletionAnnotation, + ChatCompletionAudioDelta, ChatCompletionReasoningItem, ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, @@ -1595,7 +1596,7 @@ class Delta(SafeAttributeModel, OpenAIObject): role: str | None function_call: FunctionCall | None tool_calls: list[ChatCompletionDeltaToolCall | ChatCompletionDeltaCustomToolCall] | None - audio: ChatCompletionAudioResponse | None + audio: ChatCompletionAudioResponse | ChatCompletionAudioDelta | None images: list[ImageURLListItem] | None annotations: list[ChatCompletionAnnotation] | None @@ -1610,7 +1611,7 @@ class Delta(SafeAttributeModel, OpenAIObject): role=None, function_call=None, tool_calls=None, - audio: ChatCompletionAudioResponse | None = None, + audio: ChatCompletionAudioResponse | ChatCompletionAudioDelta | None = None, images: list[ImageURLListItem] | None = None, reasoning_content: str | None = None, thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None = None, diff --git a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py index 67c78b62151..c36fbab65a1 100644 --- a/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py +++ b/tests/unit/llms/vertex_ai/gemini/test_vertex_and_google_ai_studio_gemini.py @@ -1,4 +1,4 @@ -import asyncio, importlib, os, uuid +import asyncio, base64, importlib, os, uuid import json import logging import re @@ -2130,6 +2130,59 @@ async def test_gemini_image_generation_async_stream(respx_mock: respx.MockRouter assert chunks[-1].choices[0].finish_reason == "stop" +def _gemini_tts_audio_event(audio_base64: str) -> dict[str, object]: + return { + "candidates": [ + { + "content": { + "role": "model", + "parts": [{"inlineData": {"mimeType": "audio/L16;codec=pcm;rate=24000", "data": audio_base64}}], + }, + "index": 0, + } + ], + "usageMetadata": {"promptTokenCount": 9, "totalTokenCount": 9}, + } + + +@pytest.mark.usefixtures("_gemini_httpx_transport") +@pytest.mark.respx(assert_all_called=True) +def test_gemini_tts_stream_carries_audio_in_delta_and_rebuilds_full_clip(respx_mock: respx.MockRouter) -> None: + first_audio: Final = base64.b64encode(b"\x01\x02\x03").decode() + second_audio: Final = base64.b64encode(b"\x04\x05").decode() + respx_mock.post(f"{GEMINI_MODELS_URL}/gemini-2.5-flash-preview-tts:streamGenerateContent?alt=sse").mock( + return_value=_gemini_sse( + _gemini_tts_audio_event(first_audio), + _gemini_tts_audio_event(second_audio), + { + "candidates": [ + {"content": {"role": "model", "parts": [{"text": ""}]}, "finishReason": "STOP", "index": 0} + ], + "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 40, "totalTokenCount": 49}, + }, + ) + ) + + chunks: Final = list( + litellm.completion( + model="gemini/gemini-2.5-flash-preview-tts", + messages=[{"role": "user", "content": "Say hello"}], + modalities=["audio"], + audio={"voice": "Kore", "format": "pcm16"}, + stream=True, + api_key="test-api-key", + ) + ) + complete_response: Final = litellm.stream_chunk_builder(chunks) + + assert [chunk.choices[0].delta.audio for chunk in chunks if chunk.choices[0].delta.audio is not None] == [ + {"data": first_audio}, + {"data": second_audio}, + ] + assert chunks[-1].choices[0].finish_reason == "stop" + assert base64.b64decode(complete_response.choices[0].message.audio.data) == b"\x01\x02\x03\x04\x05" + + @pytest.mark.respx(assert_all_called=True) def test_gemini_thinking(respx_mock: respx.MockRouter) -> None: route: Final = respx_mock.post(