From 900c6e5d7b6dfaec50fb2af62eb94697ea500873 Mon Sep 17 00:00:00 2001 From: AyrennC Date: Fri, 6 Jun 2025 10:48:50 -0700 Subject: [PATCH] [Docs] Add audio / tts section for gemini and vertex (#11306) * added audio and tts doc for gemini * updated gemini and vertex audio gen doc to be more concise --- docs/my-website/docs/providers/gemini.md | 114 ++++++++++++++++++++ docs/my-website/docs/providers/vertex.md | 127 +++++++++++++++++++++++ 2 files changed, 241 insertions(+) diff --git a/docs/my-website/docs/providers/gemini.md b/docs/my-website/docs/providers/gemini.md index 5d07a144514..0d388a4151f 100644 --- a/docs/my-website/docs/providers/gemini.md +++ b/docs/my-website/docs/providers/gemini.md @@ -51,6 +51,7 @@ response = completion( - frequency_penalty - modalities - reasoning_content +- audio (for TTS models only) **Anthropic Params** - thinking (used to set max budget tokens across anthropic/gemini models) @@ -201,6 +202,119 @@ curl http://0.0.0.0:4000/v1/chat/completions \ +## Text-to-Speech (TTS) Audio Output + +:::info + +LiteLLM supports Gemini TTS models that can generate audio responses using the OpenAI-compatible `audio` parameter format. + +::: + +### Supported Models + +LiteLLM supports Gemini TTS models with audio capabilities (e.g. `gemini-2.5-flash-preview-tts` and `gemini-2.5-pro-preview-tts`). For the complete list of available TTS models and voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation). + +### Limitations + +:::warning + +**Important Limitations**: +- Gemini TTS models only support the `pcm16` audio format +- **Streaming support has not been added** to TTS models yet +- The `modalities` parameter must be set to `['audio']` for TTS requests + +::: + +### Quick Start + + + + +```python +from litellm import completion +import os + +os.environ['GEMINI_API_KEY'] = "your-api-key" + +response = completion( + model="gemini/gemini-2.5-flash-preview-tts", + messages=[{"role": "user", "content": "Say hello in a friendly voice"}], + modalities=["audio"], # Required for TTS models + audio={ + "voice": "Kore", + "format": "pcm16" # Required: must be "pcm16" + } +) + +print(response) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: gemini-tts-flash + litellm_params: + model: gemini/gemini-2.5-flash-preview-tts + api_key: os.environ/GEMINI_API_KEY + - model_name: gemini-tts-pro + litellm_params: + model: gemini/gemini-2.5-pro-preview-tts + api_key: os.environ/GEMINI_API_KEY +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Make TTS request + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{ + "model": "gemini-tts-flash", + "messages": [{"role": "user", "content": "Say hello in a friendly voice"}], + "modalities": ["audio"], + "audio": { + "voice": "Kore", + "format": "pcm16" + } + }' +``` + + + + +### Advanced Usage + +You can combine TTS with other Gemini features: + +```python +response = completion( + model="gemini/gemini-2.5-pro-preview-tts", + messages=[ + {"role": "system", "content": "You are a helpful assistant that speaks clearly."}, + {"role": "user", "content": "Explain quantum computing in simple terms"} + ], + modalities=["audio"], + audio={ + "voice": "Charon", + "format": "pcm16" + }, + temperature=0.7, + max_tokens=150 +) +``` + +For more information about Gemini's TTS capabilities and available voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation). + ## Passing Gemini Specific Params ### Response schema LiteLLM supports sending `response_schema` as a param for Gemini-1.5-Pro on Google AI Studio. diff --git a/docs/my-website/docs/providers/vertex.md b/docs/my-website/docs/providers/vertex.md index 1d8e3085e3b..16c3b55d520 100644 --- a/docs/my-website/docs/providers/vertex.md +++ b/docs/my-website/docs/providers/vertex.md @@ -2841,6 +2841,133 @@ response = await litellm.aimage_generation( +## **Gemini TTS (Text-to-Speech) Audio Output** + +:::info + +LiteLLM supports Gemini TTS models on Vertex AI that can generate audio responses using the OpenAI-compatible `audio` parameter format. + +::: + +### Supported Models + +LiteLLM supports Gemini TTS models with audio capabilities on Vertex AI (e.g. `vertex_ai/gemini-2.5-flash-preview-tts` and `vertex_ai/gemini-2.5-pro-preview-tts`). For the complete list of available TTS models and voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation). + +### Limitations + +:::warning + +**Important Limitations**: +- Gemini TTS models only support the `pcm16` audio format +- **Streaming support has not been added** to TTS models yet +- The `modalities` parameter must be set to `['audio']` for TTS requests + +::: + +### Quick Start + + + + +```python +from litellm import completion +import json + +## GET CREDENTIALS +file_path = 'path/to/vertex_ai_service_account.json' + +# Load the JSON file +with open(file_path, 'r') as file: + vertex_credentials = json.load(file) + +# Convert to JSON string +vertex_credentials_json = json.dumps(vertex_credentials) + +response = completion( + model="vertex_ai/gemini-2.5-flash-preview-tts", + messages=[{"role": "user", "content": "Say hello in a friendly voice"}], + modalities=["audio"], # Required for TTS models + audio={ + "voice": "Kore", + "format": "pcm16" # Required: must be "pcm16" + }, + vertex_credentials=vertex_credentials_json +) + +print(response) +``` + + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: gemini-tts-flash + litellm_params: + model: vertex_ai/gemini-2.5-flash-preview-tts + vertex_project: "your-project-id" + vertex_location: "us-central1" + vertex_credentials: "/path/to/service_account.json" + - model_name: gemini-tts-pro + litellm_params: + model: vertex_ai/gemini-2.5-pro-preview-tts + vertex_project: "your-project-id" + vertex_location: "us-central1" + vertex_credentials: "/path/to/service_account.json" +``` + +2. Start proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Make TTS request + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer " \ + -d '{ + "model": "gemini-tts-flash", + "messages": [{"role": "user", "content": "Say hello in a friendly voice"}], + "modalities": ["audio"], + "audio": { + "voice": "Kore", + "format": "pcm16" + } + }' +``` + + + + +### Advanced Usage + +You can combine TTS with other Gemini features: + +```python +response = completion( + model="vertex_ai/gemini-2.5-pro-preview-tts", + messages=[ + {"role": "system", "content": "You are a helpful assistant that speaks clearly."}, + {"role": "user", "content": "Explain quantum computing in simple terms"} + ], + modalities=["audio"], + audio={ + "voice": "Charon", + "format": "pcm16" + }, + temperature=0.7, + max_tokens=150, + vertex_credentials=vertex_credentials_json +) +``` + +For more information about Gemini's TTS capabilities and available voices, see the [official Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/speech-generation). + ## **Text to Speech APIs** :::info