Merge pull request #5346 from BerriAI/litellm_Add_vertex_text_to_speech

[Feat-LiteLLM] Add Vertex AI - Text to speech support

docs/my-website/docs/providers/azure.md

@@ -327,6 +327,39 @@ response = litellm.completion(
 print(response)
 ```
 
+## Azure Text to Speech (tts)
+
+**LiteLLM PROXY**
+
+```yaml
+ - model_name: azure/tts-1
+    litellm_params:
+      model: azure/tts-1
+      api_base: "os.environ/AZURE_API_BASE_TTS"
+      api_key: "os.environ/AZURE_API_KEY_TTS"
+      api_version: "os.environ/AZURE_API_VERSION" 
+```
+
+**LiteLLM SDK**
+
+```python 
+from litellm import completion
+
+## set ENV variables
+os.environ["AZURE_API_KEY"] = ""
+os.environ["AZURE_API_BASE"] = ""
+os.environ["AZURE_API_VERSION"] = ""
+
+# azure call
+speech_file_path = Path(__file__).parent / "speech.mp3"
+response = speech(
+        model="azure/<your-deployment-name",
+        voice="alloy",
+        input="the quick brown fox jumped over the lazy dogs",
+    )
+response.stream_to_file(speech_file_path)
+```
+
 ## Advanced
 ### Azure API Load-Balancing
 

docs/my-website/docs/providers/vertex.md

@@ -1768,6 +1768,89 @@ response = await litellm.aimage_generation(
 )
 ```
 
+## **Text to Speech APIs**
+
+:::info
+
+LiteLLM supports calling [Vertex AI Text to Speech API](https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech) in the OpenAI text to speech API format
+
+:::
+
+
+
+Usage
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
+
+**Sync Usage**
+
+```python
+speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
+response = litellm.speech(
+    model="vertex_ai/",
+    input="hello what llm guardrail do you have",
+)
+response.stream_to_file(speech_file_path)
+```
+
+**Async Usage**
+```python
+speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
+response = litellm.aspeech(
+    model="vertex_ai/",
+    input="hello what llm guardrail do you have",
+)
+response.stream_to_file(speech_file_path)
+```
+
+</TabItem>
+<TabItem value="proxy" label="LiteLLM PROXY (Unified Endpoint)">
+
+1. Add model to config.yaml
+```yaml
+model_list:
+  - model_name: vertex-tts
+    litellm_params:
+      model: vertex_ai/ # Vertex AI does not support passing a `model` param - so passing `model=vertex_ai/` is the only required param
+      vertex_project: "adroit-crow-413218"
+      vertex_location: "us-central1"
+      vertex_credentials: adroit-crow-413218-a956eef1a2a8.json 
+
+litellm_settings:
+  drop_params: True
+```
+
+2. Start Proxy 
+
+```
+$ litellm --config /path/to/config.yaml
+```
+
+3. Make Request use OpenAI Python SDK
+
+
+```python
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+# see supported values for "voice" on vertex here: 
+# https://console.cloud.google.com/vertex-ai/generative/speech/text-to-speech
+response = client.audio.speech.create(
+    model = "vertex-tts",
+    input="the quick brown fox jumped over the lazy dogs",
+    voice={'languageCode': 'en-US', 'name': 'en-US-Studio-O'}
+)
+print("response from proxy", response)
+```
+
+</TabItem>
+</Tabs>
+
+
 ## Extra
 
 ### Using `GOOGLE_APPLICATION_CREDENTIALS`

docs/my-website/docs/text_to_speech.md

@@ -1,6 +1,7 @@
 # Text to Speech
 
-## Quick Start 
+## **LiteLLM Python SDK Usage**
+### Quick Start 
 
 ```python
 from pathlib import Path
@@ -18,7 +19,7 @@ response = speech(
 response.stream_to_file(speech_file_path)
 ```
 
-## Async Usage 
+### Async Usage 
 
 ```python
 from litellm import aspeech
@@ -47,7 +48,7 @@ async def test_async_speech():
 asyncio.run(test_async_speech())
 ```
 
-## Proxy Usage 
+## **LiteLLM Proxy Usage**
 
 LiteLLM provides an openai-compatible `/audio/speech` endpoint for Text-to-speech calls.
 
@@ -77,39 +78,13 @@ litellm --config /path/to/config.yaml
 
 # RUNNING on http://0.0.0.0:4000
 ```
+## **Supported Providers**
 
-## Azure Usage 
-
-**PROXY**
-
-```yaml
- - model_name: azure/tts-1
-    litellm_params:
-      model: azure/tts-1
-      api_base: "os.environ/AZURE_API_BASE_TTS"
-      api_key: "os.environ/AZURE_API_KEY_TTS"
-      api_version: "os.environ/AZURE_API_VERSION" 
-```
-
-**SDK**
-
-```python 
-from litellm import completion
-
-## set ENV variables
-os.environ["AZURE_API_KEY"] = ""
-os.environ["AZURE_API_BASE"] = ""
-os.environ["AZURE_API_VERSION"] = ""
-
-# azure call
-speech_file_path = Path(__file__).parent / "speech.mp3"
-response = speech(
-        model="azure/<your-deployment-name",
-        voice="alloy",
-        input="the quick brown fox jumped over the lazy dogs",
-    )
-response.stream_to_file(speech_file_path)
-```
+| Provider    | Link to Usage      |
+|-------------|--------------------|
+| OpenAI      |   [Usage](#quick-start)                 |
+| Azure OpenAI|   [Usage](../docs/providers/azure#azure-text-to-speech-tts)                 |
+| Vertex AI   |   [Usage](../docs/providers/vertex#text-to-speech-apis)                 |
 
 ## ✨ Enterprise LiteLLM Proxy - Set Max Request File Size 
 

litellm/llms/text_to_speech/vertex_ai.py

@@ -0,0 +1,203 @@
+import traceback
+from datetime import datetime
+from typing import Any, Coroutine, Literal, Optional, TypedDict, Union
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.llms.base import BaseLLM
+from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
+    HTTPHandler,
+    _get_async_httpx_client,
+    _get_httpx_client,
+)
+from litellm.llms.openai import HttpxBinaryResponseContent
+from litellm.llms.vertex_httpx import VertexLLM
+
+
+class VertexInput(TypedDict, total=False):
+    text: str
+
+
+class VertexVoice(TypedDict, total=False):
+    languageCode: str
+    name: str
+
+
+class VertexAudioConfig(TypedDict, total=False):
+    audioEncoding: str
+    speakingRate: str
+
+
+class VertexTextToSpeechRequest(TypedDict, total=False):
+    input: VertexInput
+    voice: VertexVoice
+    audioConfig: Optional[VertexAudioConfig]
+
+
+class VertexTextToSpeechAPI(VertexLLM):
+    """
+    Vertex methods to support for batches
+    """
+
+    def __init__(self) -> None:
+        super().__init__()
+
+    def audio_speech(
+        self,
+        logging_obj,
+        vertex_project: Optional[str],
+        vertex_location: Optional[str],
+        vertex_credentials: Optional[str],
+        api_base: Optional[str],
+        timeout: Union[float, httpx.Timeout],
+        model: str,
+        input: str,
+        voice: Optional[dict] = None,
+        _is_async: Optional[bool] = False,
+        optional_params: Optional[dict] = None,
+        kwargs: Optional[dict] = None,
+    ):
+        import base64
+
+        ####### Authenticate with Vertex AI ########
+        auth_header, _ = self._get_token_and_url(
+            model="",
+            gemini_api_key=None,
+            vertex_credentials=vertex_credentials,
+            vertex_project=vertex_project,
+            vertex_location=vertex_location,
+            stream=False,
+            custom_llm_provider="vertex_ai_beta",
+            api_base=api_base,
+        )
+
+        headers = {
+            "Authorization": f"Bearer {auth_header}",
+            "x-goog-user-project": vertex_project,
+            "Content-Type": "application/json",
+            "charset": "UTF-8",
+        }
+
+        ######### End of Authentication ###########
+
+        ####### Build the request ################
+        # API Ref: https://cloud.google.com/text-to-speech/docs/reference/rest/v1/text/synthesize
+        vertex_input = VertexInput(text=input)
+        # required param
+        optional_params = optional_params or {}
+        kwargs = kwargs or {}
+        if voice is not None:
+            vertex_voice = VertexVoice(**voice)
+        elif "voice" in kwargs:
+            vertex_voice = VertexVoice(**kwargs["voice"])
+        else:
+            # use defaults to not fail the request
+            vertex_voice = VertexVoice(
+                languageCode="en-US",
+                name="en-US-Studio-O",
+            )
+
+        if "audioConfig" in kwargs:
+            vertex_audio_config = VertexAudioConfig(**kwargs["audioConfig"])
+        else:
+            # use defaults to not fail the request
+            vertex_audio_config = VertexAudioConfig(
+                audioEncoding="LINEAR16",
+                speakingRate="1",
+            )
+
+        request = VertexTextToSpeechRequest(
+            input=vertex_input,
+            voice=vertex_voice,
+            audioConfig=vertex_audio_config,
+        )
+
+        url = "https://texttospeech.googleapis.com/v1/text:synthesize"
+        ########## End of building request ############
+
+        ########## Log the request for debugging / logging ############
+        logging_obj.pre_call(
+            input=[],
+            api_key="",
+            additional_args={
+                "complete_input_dict": request,
+                "api_base": url,
+                "headers": headers,
+            },
+        )
+
+        ########## End of logging ############
+        ####### Send the request ###################
+        if _is_async is True:
+            return self.async_audio_speech(
+                logging_obj=logging_obj, url=url, headers=headers, request=request
+            )
+        sync_handler = _get_httpx_client()
+
+        response = sync_handler.post(
+            url=url,
+            headers=headers,
+            json=request,  # type: ignore
+        )
+        if response.status_code != 200:
+            raise Exception(
+                f"Request failed with status code {response.status_code}, {response.text}"
+            )
+        ############ Process the response ############
+        _json_response = response.json()
+
+        response_content = _json_response["audioContent"]
+
+        # Decode base64 to get binary content
+        binary_data = base64.b64decode(response_content)
+
+        # Create an httpx.Response object
+        response = httpx.Response(
+            status_code=200,
+            content=binary_data,
+        )
+
+        # Initialize the HttpxBinaryResponseContent instance
+        http_binary_response = HttpxBinaryResponseContent(response)
+        return http_binary_response
+
+    async def async_audio_speech(
+        self,
+        logging_obj,
+        url: str,
+        headers: dict,
+        request: VertexTextToSpeechRequest,
+    ) -> HttpxBinaryResponseContent:
+        import base64
+
+        async_handler = _get_async_httpx_client()
+
+        response = await async_handler.post(
+            url=url,
+            headers=headers,
+            json=request,  # type: ignore
+        )
+
+        if response.status_code != 200:
+            raise Exception(
+                f"Request did not return a 200 status code: {response.status_code}, {response.text}"
+            )
+
+        _json_response = response.json()
+
+        response_content = _json_response["audioContent"]
+
+        # Decode base64 to get binary content
+        binary_data = base64.b64decode(response_content)
+
+        # Create an httpx.Response object
+        response = httpx.Response(
+            status_code=200,
+            content=binary_data,
+        )
+
+        # Initialize the HttpxBinaryResponseContent instance
+        http_binary_response = HttpxBinaryResponseContent(response)
+        return http_binary_response

litellm/main.py

@@ -121,6 +121,7 @@ from .llms.prompt_templates.factory import (
 )
 from .llms.sagemaker import SagemakerLLM
 from .llms.text_completion_codestral import CodestralTextCompletion
+from .llms.text_to_speech.vertex_ai import VertexTextToSpeechAPI
 from .llms.triton import TritonChatCompletion
 from .llms.vertex_ai_partner import VertexAIPartnerModels
 from .llms.vertex_httpx import VertexLLM
@@ -165,6 +166,7 @@ bedrock_chat_completion = BedrockLLM()
 bedrock_converse_chat_completion = BedrockConverseLLM()
 vertex_chat_completion = VertexLLM()
 vertex_partner_models_chat_completion = VertexAIPartnerModels()
+vertex_text_to_speech = VertexTextToSpeechAPI()
 watsonxai = IBMWatsonXAI()
 sagemaker_llm = SagemakerLLM()
 ####### COMPLETION ENDPOINTS ################
@@ -945,7 +947,6 @@ def completion(
             text_completion=kwargs.get("text_completion"),
             azure_ad_token_provider=kwargs.get("azure_ad_token_provider"),
             user_continue_message=kwargs.get("user_continue_message"),
-
         )
         logging.update_environment_variables(
             model=model,
@@ -4698,7 +4699,7 @@ async def aspeech(*args, **kwargs) -> HttpxBinaryResponseContent:
 def speech(
     model: str,
     input: str,
-    voice: str,
+    voice: Optional[Union[str, dict]] = None,
     api_key: Optional[str] = None,
     api_base: Optional[str] = None,
     api_version: Optional[str] = None,
@@ -4730,8 +4731,16 @@ def speech(
 
     if max_retries is None:
         max_retries = litellm.num_retries or openai.DEFAULT_MAX_RETRIES
+
+    logging_obj = kwargs.get("litellm_logging_obj", None)
     response: Optional[HttpxBinaryResponseContent] = None
     if custom_llm_provider == "openai":
+        if voice is None or not (isinstance(voice, str)):
+            raise litellm.BadRequestError(
+                message="'voice' is required to be passed as a string for OpenAI TTS",
+                model=model,
+                llm_provider=custom_llm_provider,
+            )
         api_base = (
             api_base  # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
             or litellm.api_base
@@ -4778,6 +4787,12 @@ def speech(
         )
     elif custom_llm_provider == "azure":
         # azure configs
+        if voice is None or not (isinstance(voice, str)):
+            raise litellm.BadRequestError(
+                message="'voice' is required to be passed as a string for Azure TTS",
+                model=model,
+                llm_provider=custom_llm_provider,
+            )
         api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE")  # type: ignore
 
         api_version = (
@@ -4815,6 +4830,46 @@ def speech(
             client=client,  # pass AsyncOpenAI, OpenAI client
             aspeech=aspeech,
         )
+    elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta":
+        from litellm.types.router import GenericLiteLLMParams
+
+        generic_optional_params = GenericLiteLLMParams(**kwargs)
+
+        api_base = generic_optional_params.api_base or ""
+        vertex_ai_project = (
+            generic_optional_params.vertex_project
+            or litellm.vertex_project
+            or get_secret("VERTEXAI_PROJECT")
+        )
+        vertex_ai_location = (
+            generic_optional_params.vertex_location
+            or litellm.vertex_location
+            or get_secret("VERTEXAI_LOCATION")
+        )
+        vertex_credentials = generic_optional_params.vertex_credentials or get_secret(
+            "VERTEXAI_CREDENTIALS"
+        )
+
+        if voice is not None and not isinstance(voice, dict):
+            raise litellm.BadRequestError(
+                message=f"'voice' is required to be passed as a dict for Vertex AI TTS, passed in voice={voice}",
+                model=model,
+                llm_provider=custom_llm_provider,
+            )
+        response = vertex_text_to_speech.audio_speech(
+            _is_async=aspeech,
+            vertex_credentials=vertex_credentials,
+            vertex_project=vertex_ai_project,
+            vertex_location=vertex_ai_location,
+            timeout=timeout,
+            api_base=api_base,
+            model=model,
+            input=input,
+            voice=voice,
+            optional_params=optional_params,
+            kwargs=kwargs,
+            logging_obj=logging_obj,
+        )
 
     if response is None:
         raise Exception(

litellm/proxy/tests/test_openai_tts_request.py

@@ -0,0 +1,11 @@
+import openai
+
+client = openai.OpenAI(api_key="sk-1234", base_url="http://0.0.0.0:4000")
+
+# # request sent to model set on litellm proxy, `litellm --model`
+response = client.audio.speech.create(
+    model="vertex-tts",
+    input="the quick brown fox jumped over the lazy dogs",
+    voice={"languageCode": "en-US", "name": "en-US-Studio-O"},  # type: ignore
+)
+print("response from proxy", response)  # noqa

litellm/tests/test_audio_speech.py

@@ -18,6 +18,7 @@ sys.path.insert(
     0, os.path.abspath("../..")
 )  # Adds the parent directory to the system path
 from pathlib import Path
+from unittest.mock import AsyncMock, MagicMock, patch
 
 import openai
 import pytest
@@ -117,3 +118,128 @@ async def test_audio_speech_router(mode):
     from litellm.llms.openai import HttpxBinaryResponseContent
 
     assert isinstance(response, HttpxBinaryResponseContent)
+
+
+@pytest.mark.parametrize(
+    "sync_mode",
+    [False, True],
+)
+@pytest.mark.skip(reason="local only test - we run testing using MockRequests below")
+@pytest.mark.asyncio
+async def test_audio_speech_litellm_vertex(sync_mode):
+    litellm.set_verbose = True
+    speech_file_path = Path(__file__).parent / "speech_vertex.mp3"
+    model = "vertex_ai/test"
+    if sync_mode:
+        response = litellm.speech(
+            model="vertex_ai/test",
+            input="hello what llm guardrail do you have",
+        )
+
+        response.stream_to_file(speech_file_path)
+
+    else:
+        response = await litellm.aspeech(
+            model="vertex_ai/",
+            input="async hello what llm guardrail do you have",
+        )
+
+        from types import SimpleNamespace
+
+        from litellm.llms.openai import HttpxBinaryResponseContent
+
+        response.stream_to_file(speech_file_path)
+
+
+@pytest.mark.asyncio
+async def test_speech_litellm_vertex_async():
+    # Mock the response
+    mock_response = AsyncMock()
+
+    def return_val():
+        return {
+            "audioContent": "dGVzdCByZXNwb25zZQ==",
+        }
+
+    mock_response.json = return_val
+    mock_response.status_code = 200
+
+    # Set up the mock for asynchronous calls
+    with patch(
+        "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
+        new_callable=AsyncMock,
+    ) as mock_async_post:
+        mock_async_post.return_value = mock_response
+        model = "vertex_ai/test"
+
+        response = await litellm.aspeech(
+            model=model,
+            input="async hello what llm guardrail do you have",
+        )
+
+        # Assert asynchronous call
+        mock_async_post.assert_called_once()
+        _, kwargs = mock_async_post.call_args
+        print("call args", kwargs)
+
+        assert kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize"
+
+        assert "x-goog-user-project" in kwargs["headers"]
+        assert kwargs["headers"]["Authorization"] is not None
+
+        assert kwargs["json"] == {
+            "input": {"text": "async hello what llm guardrail do you have"},
+            "voice": {"languageCode": "en-US", "name": "en-US-Studio-O"},
+            "audioConfig": {"audioEncoding": "LINEAR16", "speakingRate": "1"},
+        }
+
+
+@pytest.mark.asyncio
+async def test_speech_litellm_vertex_async_with_voice():
+    # Mock the response
+    mock_response = AsyncMock()
+
+    def return_val():
+        return {
+            "audioContent": "dGVzdCByZXNwb25zZQ==",
+        }
+
+    mock_response.json = return_val
+    mock_response.status_code = 200
+
+    # Set up the mock for asynchronous calls
+    with patch(
+        "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
+        new_callable=AsyncMock,
+    ) as mock_async_post:
+        mock_async_post.return_value = mock_response
+        model = "vertex_ai/test"
+
+        response = await litellm.aspeech(
+            model=model,
+            input="async hello what llm guardrail do you have",
+            voice={
+                "languageCode": "en-UK",
+                "name": "en-UK-Studio-O",
+            },
+            audioConfig={
+                "audioEncoding": "LINEAR22",
+                "speakingRate": "10",
+            },
+        )
+
+        # Assert asynchronous call
+        mock_async_post.assert_called_once()
+        _, kwargs = mock_async_post.call_args
+        print("call args", kwargs)
+
+        assert kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize"
+
+        assert "x-goog-user-project" in kwargs["headers"]
+        assert kwargs["headers"]["Authorization"] is not None
+
+        assert kwargs["json"] == {
+            "input": {"text": "async hello what llm guardrail do you have"},
+            "voice": {"languageCode": "en-UK", "name": "en-UK-Studio-O"},
+            "audioConfig": {"audioEncoding": "LINEAR22", "speakingRate": "10"},
+        }