fix(gemini): stream TTS audio in delta.audio on chat completions

This commit is contained in:
mateo-berri 2026-10-30 01:56:20 -07:00
parent 73aaf9e3da
commit e3471788f2
3 changed files with 62 additions and 3 deletions

View file

@ -49,6 +49,7 @@ from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionAnnotation,
ChatCompletionAnnotationURLCitation,
ChatCompletionAudioDelta,
ChatCompletionResponseMessage,
ChatCompletionThinkingBlock,
ChatCompletionToolCallChunk,
@ -2568,6 +2569,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
functions: ChatCompletionToolCallFunctionChunk | None,
chat_completion_logprobs: ChoiceLogprobs | None,
image_response: list[ImageURLListItem] | None,
audio_response: ChatCompletionAudioResponse | None = None,
) -> StreamingChoices:
"""
Helper method to create a streaming choice object for Vertex AI
@ -2586,6 +2588,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
content=chat_completion_message.get("content"),
reasoning_content=chat_completion_message.get("reasoning_content"),
tool_calls=tools,
audio=ChatCompletionAudioDelta(data=audio_response.data) if audio_response is not None else None,
images=image_response,
function_call=functions,
annotations=annotations,
@ -2795,6 +2798,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
continue
image_response: list[ImageURLListItem] | None = None
audio_response: ChatCompletionAudioResponse | None = None
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
chat_completion_logprobs: ChoiceLogprobs | None = None
tools: list[ChatCompletionToolCallChunk] | None = None
@ -2921,6 +2925,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
functions=functions,
chat_completion_logprobs=chat_completion_logprobs,
image_response=image_response,
audio_response=audio_response,
)
model_response.choices.append(choice)
elif isinstance(model_response, ModelResponse):

View file

@ -86,6 +86,7 @@ from .llms.openai import (
AllMessageValues,
Batch,
ChatCompletionAnnotation,
ChatCompletionAudioDelta,
ChatCompletionReasoningItem,
ChatCompletionRedactedThinkingBlock,
ChatCompletionThinkingBlock,
@ -1595,7 +1596,7 @@ class Delta(SafeAttributeModel, OpenAIObject):
role: str | None
function_call: FunctionCall | None
tool_calls: list[ChatCompletionDeltaToolCall | ChatCompletionDeltaCustomToolCall] | None
audio: ChatCompletionAudioResponse | None
audio: ChatCompletionAudioResponse | ChatCompletionAudioDelta | None
images: list[ImageURLListItem] | None
annotations: list[ChatCompletionAnnotation] | None
@ -1610,7 +1611,7 @@ class Delta(SafeAttributeModel, OpenAIObject):
role=None,
function_call=None,
tool_calls=None,
audio: ChatCompletionAudioResponse | None = None,
audio: ChatCompletionAudioResponse | ChatCompletionAudioDelta | None = None,
images: list[ImageURLListItem] | None = None,
reasoning_content: str | None = None,
thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None = None,

View file

@ -1,4 +1,4 @@
import asyncio, importlib, os, uuid
import asyncio, base64, importlib, os, uuid
import json
import logging
import re
@ -2130,6 +2130,59 @@ async def test_gemini_image_generation_async_stream(respx_mock: respx.MockRouter
assert chunks[-1].choices[0].finish_reason == "stop"
def _gemini_tts_audio_event(audio_base64: str) -> dict[str, object]:
return {
"candidates": [
{
"content": {
"role": "model",
"parts": [{"inlineData": {"mimeType": "audio/L16;codec=pcm;rate=24000", "data": audio_base64}}],
},
"index": 0,
}
],
"usageMetadata": {"promptTokenCount": 9, "totalTokenCount": 9},
}
@pytest.mark.usefixtures("_gemini_httpx_transport")
@pytest.mark.respx(assert_all_called=True)
def test_gemini_tts_stream_carries_audio_in_delta_and_rebuilds_full_clip(respx_mock: respx.MockRouter) -> None:
first_audio: Final = base64.b64encode(b"\x01\x02\x03").decode()
second_audio: Final = base64.b64encode(b"\x04\x05").decode()
respx_mock.post(f"{GEMINI_MODELS_URL}/gemini-2.5-flash-preview-tts:streamGenerateContent?alt=sse").mock(
return_value=_gemini_sse(
_gemini_tts_audio_event(first_audio),
_gemini_tts_audio_event(second_audio),
{
"candidates": [
{"content": {"role": "model", "parts": [{"text": ""}]}, "finishReason": "STOP", "index": 0}
],
"usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 40, "totalTokenCount": 49},
},
)
)
chunks: Final = list(
litellm.completion(
model="gemini/gemini-2.5-flash-preview-tts",
messages=[{"role": "user", "content": "Say hello"}],
modalities=["audio"],
audio={"voice": "Kore", "format": "pcm16"},
stream=True,
api_key="test-api-key",
)
)
complete_response: Final = litellm.stream_chunk_builder(chunks)
assert [chunk.choices[0].delta.audio for chunk in chunks if chunk.choices[0].delta.audio is not None] == [
{"data": first_audio},
{"data": second_audio},
]
assert chunks[-1].choices[0].finish_reason == "stop"
assert base64.b64decode(complete_response.choices[0].message.audio.data) == b"\x01\x02\x03\x04\x05"
@pytest.mark.respx(assert_all_called=True)
def test_gemini_thinking(respx_mock: respx.MockRouter) -> None:
route: Final = respx_mock.post(