mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(gemini): stream TTS audio in delta.audio on chat completions
This commit is contained in:
parent
73aaf9e3da
commit
e3471788f2
3 changed files with 62 additions and 3 deletions
|
|
@ -49,6 +49,7 @@ from litellm.types.llms.openai import (
|
|||
AllMessageValues,
|
||||
ChatCompletionAnnotation,
|
||||
ChatCompletionAnnotationURLCitation,
|
||||
ChatCompletionAudioDelta,
|
||||
ChatCompletionResponseMessage,
|
||||
ChatCompletionThinkingBlock,
|
||||
ChatCompletionToolCallChunk,
|
||||
|
|
@ -2568,6 +2569,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
functions: ChatCompletionToolCallFunctionChunk | None,
|
||||
chat_completion_logprobs: ChoiceLogprobs | None,
|
||||
image_response: list[ImageURLListItem] | None,
|
||||
audio_response: ChatCompletionAudioResponse | None = None,
|
||||
) -> StreamingChoices:
|
||||
"""
|
||||
Helper method to create a streaming choice object for Vertex AI
|
||||
|
|
@ -2586,6 +2588,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
content=chat_completion_message.get("content"),
|
||||
reasoning_content=chat_completion_message.get("reasoning_content"),
|
||||
tool_calls=tools,
|
||||
audio=ChatCompletionAudioDelta(data=audio_response.data) if audio_response is not None else None,
|
||||
images=image_response,
|
||||
function_call=functions,
|
||||
annotations=annotations,
|
||||
|
|
@ -2795,6 +2798,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
continue
|
||||
|
||||
image_response: list[ImageURLListItem] | None = None
|
||||
audio_response: ChatCompletionAudioResponse | None = None
|
||||
chat_completion_message: ChatCompletionResponseMessage = {"role": "assistant"}
|
||||
chat_completion_logprobs: ChoiceLogprobs | None = None
|
||||
tools: list[ChatCompletionToolCallChunk] | None = None
|
||||
|
|
@ -2921,6 +2925,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
functions=functions,
|
||||
chat_completion_logprobs=chat_completion_logprobs,
|
||||
image_response=image_response,
|
||||
audio_response=audio_response,
|
||||
)
|
||||
model_response.choices.append(choice)
|
||||
elif isinstance(model_response, ModelResponse):
|
||||
|
|
|
|||
|
|
@ -86,6 +86,7 @@ from .llms.openai import (
|
|||
AllMessageValues,
|
||||
Batch,
|
||||
ChatCompletionAnnotation,
|
||||
ChatCompletionAudioDelta,
|
||||
ChatCompletionReasoningItem,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
ChatCompletionThinkingBlock,
|
||||
|
|
@ -1595,7 +1596,7 @@ class Delta(SafeAttributeModel, OpenAIObject):
|
|||
role: str | None
|
||||
function_call: FunctionCall | None
|
||||
tool_calls: list[ChatCompletionDeltaToolCall | ChatCompletionDeltaCustomToolCall] | None
|
||||
audio: ChatCompletionAudioResponse | None
|
||||
audio: ChatCompletionAudioResponse | ChatCompletionAudioDelta | None
|
||||
images: list[ImageURLListItem] | None
|
||||
annotations: list[ChatCompletionAnnotation] | None
|
||||
|
||||
|
|
@ -1610,7 +1611,7 @@ class Delta(SafeAttributeModel, OpenAIObject):
|
|||
role=None,
|
||||
function_call=None,
|
||||
tool_calls=None,
|
||||
audio: ChatCompletionAudioResponse | None = None,
|
||||
audio: ChatCompletionAudioResponse | ChatCompletionAudioDelta | None = None,
|
||||
images: list[ImageURLListItem] | None = None,
|
||||
reasoning_content: str | None = None,
|
||||
thinking_blocks: list[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock] | None = None,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import asyncio, importlib, os, uuid
|
||||
import asyncio, base64, importlib, os, uuid
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
|
@ -2130,6 +2130,59 @@ async def test_gemini_image_generation_async_stream(respx_mock: respx.MockRouter
|
|||
assert chunks[-1].choices[0].finish_reason == "stop"
|
||||
|
||||
|
||||
def _gemini_tts_audio_event(audio_base64: str) -> dict[str, object]:
|
||||
return {
|
||||
"candidates": [
|
||||
{
|
||||
"content": {
|
||||
"role": "model",
|
||||
"parts": [{"inlineData": {"mimeType": "audio/L16;codec=pcm;rate=24000", "data": audio_base64}}],
|
||||
},
|
||||
"index": 0,
|
||||
}
|
||||
],
|
||||
"usageMetadata": {"promptTokenCount": 9, "totalTokenCount": 9},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_gemini_httpx_transport")
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_gemini_tts_stream_carries_audio_in_delta_and_rebuilds_full_clip(respx_mock: respx.MockRouter) -> None:
|
||||
first_audio: Final = base64.b64encode(b"\x01\x02\x03").decode()
|
||||
second_audio: Final = base64.b64encode(b"\x04\x05").decode()
|
||||
respx_mock.post(f"{GEMINI_MODELS_URL}/gemini-2.5-flash-preview-tts:streamGenerateContent?alt=sse").mock(
|
||||
return_value=_gemini_sse(
|
||||
_gemini_tts_audio_event(first_audio),
|
||||
_gemini_tts_audio_event(second_audio),
|
||||
{
|
||||
"candidates": [
|
||||
{"content": {"role": "model", "parts": [{"text": ""}]}, "finishReason": "STOP", "index": 0}
|
||||
],
|
||||
"usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 40, "totalTokenCount": 49},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
chunks: Final = list(
|
||||
litellm.completion(
|
||||
model="gemini/gemini-2.5-flash-preview-tts",
|
||||
messages=[{"role": "user", "content": "Say hello"}],
|
||||
modalities=["audio"],
|
||||
audio={"voice": "Kore", "format": "pcm16"},
|
||||
stream=True,
|
||||
api_key="test-api-key",
|
||||
)
|
||||
)
|
||||
complete_response: Final = litellm.stream_chunk_builder(chunks)
|
||||
|
||||
assert [chunk.choices[0].delta.audio for chunk in chunks if chunk.choices[0].delta.audio is not None] == [
|
||||
{"data": first_audio},
|
||||
{"data": second_audio},
|
||||
]
|
||||
assert chunks[-1].choices[0].finish_reason == "stop"
|
||||
assert base64.b64decode(complete_response.choices[0].message.audio.data) == b"\x01\x02\x03\x04\x05"
|
||||
|
||||
|
||||
@pytest.mark.respx(assert_all_called=True)
|
||||
def test_gemini_thinking(respx_mock: respx.MockRouter) -> None:
|
||||
route: Final = respx_mock.post(
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue