diff --git a/litellm/llms/flowspeech/__init__.py b/litellm/llms/flowspeech/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/litellm/llms/flowspeech/__init__.py @@ -0,0 +1 @@ + diff --git a/litellm/llms/flowspeech/text_to_speech/__init__.py b/litellm/llms/flowspeech/text_to_speech/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/litellm/llms/flowspeech/text_to_speech/__init__.py @@ -0,0 +1 @@ + diff --git a/litellm/llms/flowspeech/text_to_speech/transformation.py b/litellm/llms/flowspeech/text_to_speech/transformation.py new file mode 100644 index 00000000000..5d3f9847017 --- /dev/null +++ b/litellm/llms/flowspeech/text_to_speech/transformation.py @@ -0,0 +1,207 @@ +import base64 +import binascii +from typing import TYPE_CHECKING, Final + +import httpx +from httpx import Headers +from pydantic import BaseModel + +import litellm +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.text_to_speech.transformation import ( + BaseTextToSpeechConfig, + TextToSpeechRequestData, +) +from litellm.secret_managers.main import get_secret_str + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.types.llms.openai import HttpxBinaryResponseContent + + +class FlowSpeechException(BaseLLMException): + def __init__( + self, + status_code: int, + message: str, + headers: dict[str, str] | Headers | None = None, # mutable-ok: exception contract accepts dict headers + ) -> None: + super().__init__( # pyright: ignore[reportUnknownMemberType] # upstream exception keeps untyped dict fields + status_code=status_code, + message=message, + headers=headers, + ) + + +class FlowSpeechAudioData(BaseModel): + mimeType: str + audioBase64: str + + +class FlowSpeechResponse(BaseModel): + code: int + message: str | None = None + data: FlowSpeechAudioData | None = None + + +class FlowSpeechTextToSpeechConfig(BaseTextToSpeechConfig): + TTS_BASE_URL = "https://flowspeech.io" + TTS_ENDPOINT_PATH = "/api/ai/text-to-speech" + DEFAULT_VOICE = "Kore" + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: base provider interface returns list + return ["voice", "instructions"] # mutable-ok: base provider interface returns list + + def map_openai_params( + self, + model: str, + optional_params: dict[str, object], # mutable-ok: base provider interface requires dict + voice: str | dict[str, object] | None = None, # mutable-ok: base provider interface accepts dict voices + drop_params: bool = False, + kwargs: dict[str, object] | None = None, # mutable-ok: base provider interface requires dict + ) -> tuple[str | None, dict[str, object]]: # mutable-ok: base provider interface returns dict params + params: Final[dict[str, object]] = ( # mutable-ok: isolated request copy + dict(optional_params) if optional_params else {} # mutable-ok: isolated request copy + ) + mapped_voice: Final = self._resolve_voice(voice) + instructions: Final = params.get("instructions") + mapped_params: Final[dict[str, object]] = ( # mutable-ok: base provider interface returns dict params + {"prompt": instructions} # mutable-ok: base provider interface returns dict params + if isinstance(instructions, str) and instructions.strip() + else {} # mutable-ok: base provider interface returns dict params + ) + return mapped_voice, mapped_params + + def _resolve_voice( + self, + voice: str | dict[str, object] | None, # mutable-ok: base provider interface accepts dict voices + ) -> str: + if isinstance(voice, str) and voice.strip(): + return voice.strip() + if isinstance(voice, dict): + candidate: Final = next( + ( + value + for key in ("voice_name", "voiceName", "id", "name") + if isinstance((value := voice.get(key)), str) and value.strip() + ), + None, + ) + if isinstance(candidate, str): + return candidate.strip() + return self.DEFAULT_VOICE + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: base provider interface requires dict headers + model: str, + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, str]: # mutable-ok: base provider interface returns dict headers + resolved_api_key: Final = api_key or litellm.api_key or get_secret_str("FLOWSPEECH_API_KEY") + if resolved_api_key is None: + raise ValueError("FlowSpeech API key is required. Set FLOWSPEECH_API_KEY environment variable.") + return { # mutable-ok: base provider interface returns dict headers + **headers, + "Authorization": f"Bearer {resolved_api_key}", + "Content-Type": "application/json", + } + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | Headers, # mutable-ok: exception contract accepts dict headers + ) -> BaseLLMException: + return FlowSpeechException(message=error_message, status_code=status_code, headers=headers) + + def transform_text_to_speech_request( + self, + model: str, + input: str, + voice: str | None, + optional_params: dict[str, object], # mutable-ok: base provider interface requires dict + litellm_params: dict[str, object], # mutable-ok: base provider interface requires dict + headers: dict[str, str], # mutable-ok: base provider interface requires dict + ) -> TextToSpeechRequestData: + prompt: Final = optional_params.get("prompt") + prompt_data: Final[dict[str, str]] = ( # mutable-ok: JSON request body requires a dict + {"prompt": prompt} # mutable-ok: JSON request body requires a dict + if isinstance(prompt, str) and prompt.strip() + else {} # mutable-ok: JSON request body requires a dict + ) + request_body: Final[dict[str, object]] = { # mutable-ok: JSON request body requires a dict + "text": input, + "originalText": input, + "speakers": [ # mutable-ok: FlowSpeech JSON schema requires a speakers array + {"voiceName": voice or self.DEFAULT_VOICE} # mutable-ok: FlowSpeech JSON schema requires an object + ], + **prompt_data, + } + return TextToSpeechRequestData( + dict_body=request_body, + headers={"Content-Type": "application/json"}, # mutable-ok: base request type requires dict headers + ) + + def transform_text_to_speech_response( + self, + model: str, + raw_response: httpx.Response, + logging_obj: "LiteLLMLoggingObj", + ) -> "HttpxBinaryResponseContent": + from litellm.types.llms.openai import HttpxBinaryResponseContent + + try: + payload: Final = FlowSpeechResponse.model_validate(raw_response.json()) + except ValueError as exc: + raise FlowSpeechException( + status_code=raw_response.status_code, + message="FlowSpeech API returned an invalid JSON response", + headers=raw_response.headers, + ) from exc + + if payload.code != 0 or payload.data is None or not payload.data.audioBase64: + raise FlowSpeechException( + status_code=raw_response.status_code, + message=payload.message or "FlowSpeech API response did not include audio data", + headers=raw_response.headers, + ) + + try: + audio_bytes: Final = base64.b64decode(payload.data.audioBase64, validate=True) + except (binascii.Error, ValueError) as exc: + raise FlowSpeechException( + status_code=raw_response.status_code, + message="FlowSpeech API returned invalid base64 audio data", + headers=raw_response.headers, + ) from exc + + preserved_headers: Final = { # mutable-ok: httpx.Response requires materialized headers + key: value + for key, value in raw_response.headers.items() + if key.lower() not in frozenset({"content-encoding", "content-length", "content-type", "transfer-encoding"}) + } + response_headers: Final = { # mutable-ok: httpx.Response requires materialized headers + **preserved_headers, + "content-length": str(len(audio_bytes)), + "content-type": payload.data.mimeType, + } + binary_response: Final = httpx.Response( + status_code=200, + headers=response_headers, + content=audio_bytes, + request=raw_response.request, + ) + return HttpxBinaryResponseContent(binary_response) + + def get_complete_url( + self, + model: str, + api_base: str | None, + litellm_params: dict[str, object], # mutable-ok: base provider interface requires dict + ) -> str: + base_url: Final = api_base or get_secret_str("FLOWSPEECH_API_BASE") or self.TTS_BASE_URL + normalized_base_url: Final = base_url.rstrip("/") + if normalized_base_url.endswith(self.TTS_ENDPOINT_PATH): + return normalized_base_url + return f"{normalized_base_url}{self.TTS_ENDPOINT_PATH}" diff --git a/litellm/main.py b/litellm/main.py index 8ee102f5d07..e9c1b9fa281 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -8225,6 +8225,40 @@ def speech( client=client, _is_async=aspeech or False, ) + elif custom_llm_provider == "flowspeech": + from litellm.llms.flowspeech.text_to_speech.transformation import ( + FlowSpeechTextToSpeechConfig, + ) + + if text_to_speech_provider_config is None: + text_to_speech_provider_config = ( # rebind-ok: provider config is resolved lazily for speech dispatch + FlowSpeechTextToSpeechConfig() + ) + + if not isinstance(text_to_speech_provider_config, FlowSpeechTextToSpeechConfig): + raise TypeError("FlowSpeech TTS configuration has an unexpected type") + flowspeech_config: Final = text_to_speech_provider_config + + if api_base is not None: + litellm_params_dict["api_base"] = api_base + if api_key is not None: + litellm_params_dict["api_key"] = api_key + + voice_name: Final = voice if isinstance(voice, str) else None + response = base_llm_http_handler.text_to_speech_handler( # rebind-ok: provider branch sets dispatch result + model=model, + input=input, + voice=voice_name, + text_to_speech_provider_config=flowspeech_config, + text_to_speech_optional_params=optional_params, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params_dict, + logging_obj=logging_obj, + timeout=timeout, + extra_headers=extra_headers, + client=client, + _is_async=aspeech or False, + ) elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta": from litellm.llms.vertex_ai.text_to_speech.transformation import ( VertexAITextToSpeechConfig, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 58103b84749..092a405a180 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3783,6 +3783,7 @@ class LlmProviders(str, Enum): INFINITY = "infinity" DEEPGRAM = "deepgram" ELEVENLABS = "elevenlabs" + FLOWSPEECH = "flowspeech" NOVITA = "novita" AIOHTTP_OPENAI = "aiohttp_openai" LANGFUSE = "langfuse" diff --git a/litellm/utils.py b/litellm/utils.py index 54f97ccae54..85da141dab1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -9309,6 +9309,12 @@ class ProviderConfigManager: ) return ElevenLabsTextToSpeechConfig() + elif litellm.LlmProviders.FLOWSPEECH == provider: + from litellm.llms.flowspeech.text_to_speech.transformation import ( + FlowSpeechTextToSpeechConfig, + ) + + return FlowSpeechTextToSpeechConfig() elif litellm.LlmProviders.RUNWAYML == provider: from litellm.llms.runwayml.text_to_speech.transformation import ( RunwayMLTextToSpeechConfig, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index dd367e875de..8a543eddcd4 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -17298,6 +17298,14 @@ "/v1/audio/transcriptions" ] }, + "flowspeech/flowspeech-tts": { + "litellm_provider": "flowspeech", + "mode": "audio_speech", + "source": "https://flowspeech.io/", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, "elevenlabs/eleven_v3": { "input_cost_per_character": 0.00018, "litellm_provider": "elevenlabs", diff --git a/tests/test_litellm/llms/flowspeech/__init__.py b/tests/test_litellm/llms/flowspeech/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/test_litellm/llms/flowspeech/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_litellm/llms/flowspeech/test_text_to_speech_transformation.py b/tests/test_litellm/llms/flowspeech/test_text_to_speech_transformation.py new file mode 100644 index 00000000000..903495a19d4 --- /dev/null +++ b/tests/test_litellm/llms/flowspeech/test_text_to_speech_transformation.py @@ -0,0 +1,168 @@ +import base64 +import json + +import httpx +import pytest + +import litellm +from litellm.llms.custom_httpx.http_handler import HTTPHandler +from litellm.llms.flowspeech.text_to_speech.transformation import ( + FlowSpeechException, + FlowSpeechTextToSpeechConfig, +) + + +def test_maps_voice_and_instructions_to_flowspeech_request(): + config = FlowSpeechTextToSpeechConfig() + voice, params = config.map_openai_params( + model="flowspeech-tts", + optional_params={"instructions": "Speak with calm confidence"}, + voice={"voiceName": "Aoede"}, + ) + + request = config.transform_text_to_speech_request( + model="flowspeech-tts", + input="Hello from FlowSpeech", + voice=voice, + optional_params=params, + litellm_params={}, + headers={}, + ) + + assert request["dict_body"] == { + "text": "Hello from FlowSpeech", + "originalText": "Hello from FlowSpeech", + "speakers": [{"voiceName": "Aoede"}], + "prompt": "Speak with calm confidence", + } + + +def test_uses_default_voice_and_ignores_unsupported_openai_params(): + config = FlowSpeechTextToSpeechConfig() + voice, params = config.map_openai_params( + model="flowspeech-tts", + optional_params={"response_format": "mp3", "speed": 1.5}, + ) + + assert voice == "Kore" + assert params == {} + + +def test_validates_bearer_api_key(): + config = FlowSpeechTextToSpeechConfig() + + headers = config.validate_environment({}, "flowspeech-tts", api_key="test-key") + + assert headers == { + "Authorization": "Bearer test-key", + "Content-Type": "application/json", + } + + +def test_requires_api_key(monkeypatch): + config = FlowSpeechTextToSpeechConfig() + monkeypatch.setattr(litellm, "api_key", None) + monkeypatch.delenv("FLOWSPEECH_API_KEY", raising=False) + + with pytest.raises(ValueError, match="FlowSpeech API key is required"): + config.validate_environment({}, "flowspeech-tts") + + +def test_decodes_flowspeech_audio_response(): + config = FlowSpeechTextToSpeechConfig() + audio = b"test audio bytes" + response = httpx.Response( + 200, + headers={"content-type": "application/json", "x-request-id": "request-1"}, + json={ + "code": 0, + "data": { + "mimeType": "audio/mpeg", + "audioBase64": base64.b64encode(audio).decode(), + }, + }, + request=httpx.Request("POST", "https://flowspeech.io/api/ai/text-to-speech"), + ) + + result = config.transform_text_to_speech_response("flowspeech-tts", response, logging_obj=None) + + assert result.content == audio + assert result.response.headers["content-type"] == "audio/mpeg" + assert result.response.headers["x-request-id"] == "request-1" + + +def test_rejects_api_error_response(): + config = FlowSpeechTextToSpeechConfig() + response = httpx.Response( + 400, + json={"code": 4001, "message": "Quota exceeded", "data": None}, + request=httpx.Request("POST", "https://flowspeech.io/api/ai/text-to-speech"), + ) + + with pytest.raises(FlowSpeechException, match="Quota exceeded"): + config.transform_text_to_speech_response("flowspeech-tts", response, logging_obj=None) + + +def test_builds_default_and_custom_urls(): + config = FlowSpeechTextToSpeechConfig() + + assert config.get_complete_url("flowspeech-tts", None, {}) == "https://flowspeech.io/api/ai/text-to-speech" + assert ( + config.get_complete_url( + "flowspeech-tts", + "https://example.com/api/ai/text-to-speech", + {}, + ) + == "https://example.com/api/ai/text-to-speech" + ) + + +def test_registers_flowspeech_provider_and_config(): + from litellm.utils import ProviderConfigManager + + model, provider, _, _ = litellm.get_llm_provider("flowspeech/flowspeech-tts") + config = ProviderConfigManager.get_provider_text_to_speech_config( + model=model, + provider=litellm.LlmProviders.FLOWSPEECH, + ) + + assert model == "flowspeech-tts" + assert provider == "flowspeech" + assert isinstance(config, FlowSpeechTextToSpeechConfig) + + +def test_speech_dispatches_to_flowspeech_handler(): + audio = b"generated audio" + + def respond(request: httpx.Request) -> httpx.Response: + assert request.url == "https://flowspeech.io/api/ai/text-to-speech" + assert request.headers["authorization"] == "Bearer test-key" + assert json.loads(request.content) == { + "text": "Hello", + "originalText": "Hello", + "speakers": [{"voiceName": "Aoede"}], + "prompt": "Speak warmly", + } + return httpx.Response( + 200, + json={ + "code": 0, + "data": { + "mimeType": "audio/mpeg", + "audioBase64": base64.b64encode(audio).decode(), + }, + }, + ) + + client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + result = litellm.speech( + model="flowspeech/flowspeech-tts", + input="Hello", + voice="Aoede", + instructions="Speak warmly", + api_key="test-key", + client=client, + ) + + assert result.content == audio + assert result.response.headers["content-type"] == "audio/mpeg"