mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-10 22:41:41 +00:00
feat: add FlowSpeech text-to-speech provider
This commit is contained in:
heng.yin
parent
cd63c7e5a7
commit
f801354c70
9 changed files with 427 additions and 0 deletions
1
litellm/llms/flowspeech/__init__.py
Normal file
1
litellm/llms/flowspeech/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
|
||||
1
litellm/llms/flowspeech/text_to_speech/__init__.py
Normal file
1
litellm/llms/flowspeech/text_to_speech/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
|
||||
207
litellm/llms/flowspeech/text_to_speech/transformation.py
Normal file
207
litellm/llms/flowspeech/text_to_speech/transformation.py
Normal file
|
|
@ -0,0 +1,207 @@
|
|||
import base64
|
||||
import binascii
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from httpx import Headers
|
||||
from pydantic import BaseModel
|
||||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.llms.base_llm.text_to_speech.transformation import (
|
||||
BaseTextToSpeechConfig,
|
||||
TextToSpeechRequestData,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
|
||||
class FlowSpeechException(BaseLLMException):
|
||||
def __init__(
|
||||
self,
|
||||
status_code: int,
|
||||
message: str,
|
||||
headers: dict[str, str] | Headers | None = None, # mutable-ok: exception contract accepts dict headers
|
||||
) -> None:
|
||||
super().__init__( # pyright: ignore[reportUnknownMemberType] # upstream exception keeps untyped dict fields
|
||||
status_code=status_code,
|
||||
message=message,
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
|
||||
class FlowSpeechAudioData(BaseModel):
|
||||
mimeType: str
|
||||
audioBase64: str
|
||||
|
||||
|
||||
class FlowSpeechResponse(BaseModel):
|
||||
code: int
|
||||
message: str | None = None
|
||||
data: FlowSpeechAudioData | None = None
|
||||
|
||||
|
||||
class FlowSpeechTextToSpeechConfig(BaseTextToSpeechConfig):
|
||||
TTS_BASE_URL = "https://flowspeech.io"
|
||||
TTS_ENDPOINT_PATH = "/api/ai/text-to-speech"
|
||||
DEFAULT_VOICE = "Kore"
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: base provider interface returns list
|
||||
return ["voice", "instructions"] # mutable-ok: base provider interface returns list
|
||||
|
||||
def map_openai_params(
|
||||
self,
|
||||
model: str,
|
||||
optional_params: dict[str, object], # mutable-ok: base provider interface requires dict
|
||||
voice: str | dict[str, object] | None = None, # mutable-ok: base provider interface accepts dict voices
|
||||
drop_params: bool = False,
|
||||
kwargs: dict[str, object] | None = None, # mutable-ok: base provider interface requires dict
|
||||
) -> tuple[str | None, dict[str, object]]: # mutable-ok: base provider interface returns dict params
|
||||
params: Final[dict[str, object]] = ( # mutable-ok: isolated request copy
|
||||
dict(optional_params) if optional_params else {} # mutable-ok: isolated request copy
|
||||
)
|
||||
mapped_voice: Final = self._resolve_voice(voice)
|
||||
instructions: Final = params.get("instructions")
|
||||
mapped_params: Final[dict[str, object]] = ( # mutable-ok: base provider interface returns dict params
|
||||
{"prompt": instructions} # mutable-ok: base provider interface returns dict params
|
||||
if isinstance(instructions, str) and instructions.strip()
|
||||
else {} # mutable-ok: base provider interface returns dict params
|
||||
)
|
||||
return mapped_voice, mapped_params
|
||||
|
||||
def _resolve_voice(
|
||||
self,
|
||||
voice: str | dict[str, object] | None, # mutable-ok: base provider interface accepts dict voices
|
||||
) -> str:
|
||||
if isinstance(voice, str) and voice.strip():
|
||||
return voice.strip()
|
||||
if isinstance(voice, dict):
|
||||
candidate: Final = next(
|
||||
(
|
||||
value
|
||||
for key in ("voice_name", "voiceName", "id", "name")
|
||||
if isinstance((value := voice.get(key)), str) and value.strip()
|
||||
),
|
||||
None,
|
||||
)
|
||||
if isinstance(candidate, str):
|
||||
return candidate.strip()
|
||||
return self.DEFAULT_VOICE
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict[str, str], # mutable-ok: base provider interface requires dict headers
|
||||
model: str,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict[str, str]: # mutable-ok: base provider interface returns dict headers
|
||||
resolved_api_key: Final = api_key or litellm.api_key or get_secret_str("FLOWSPEECH_API_KEY")
|
||||
if resolved_api_key is None:
|
||||
raise ValueError("FlowSpeech API key is required. Set FLOWSPEECH_API_KEY environment variable.")
|
||||
return { # mutable-ok: base provider interface returns dict headers
|
||||
**headers,
|
||||
"Authorization": f"Bearer {resolved_api_key}",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: dict[str, str] | Headers, # mutable-ok: exception contract accepts dict headers
|
||||
) -> BaseLLMException:
|
||||
return FlowSpeechException(message=error_message, status_code=status_code, headers=headers)
|
||||
|
||||
def transform_text_to_speech_request(
|
||||
self,
|
||||
model: str,
|
||||
input: str,
|
||||
voice: str | None,
|
||||
optional_params: dict[str, object], # mutable-ok: base provider interface requires dict
|
||||
litellm_params: dict[str, object], # mutable-ok: base provider interface requires dict
|
||||
headers: dict[str, str], # mutable-ok: base provider interface requires dict
|
||||
) -> TextToSpeechRequestData:
|
||||
prompt: Final = optional_params.get("prompt")
|
||||
prompt_data: Final[dict[str, str]] = ( # mutable-ok: JSON request body requires a dict
|
||||
{"prompt": prompt} # mutable-ok: JSON request body requires a dict
|
||||
if isinstance(prompt, str) and prompt.strip()
|
||||
else {} # mutable-ok: JSON request body requires a dict
|
||||
)
|
||||
request_body: Final[dict[str, object]] = { # mutable-ok: JSON request body requires a dict
|
||||
"text": input,
|
||||
"originalText": input,
|
||||
"speakers": [ # mutable-ok: FlowSpeech JSON schema requires a speakers array
|
||||
{"voiceName": voice or self.DEFAULT_VOICE} # mutable-ok: FlowSpeech JSON schema requires an object
|
||||
],
|
||||
**prompt_data,
|
||||
}
|
||||
return TextToSpeechRequestData(
|
||||
dict_body=request_body,
|
||||
headers={"Content-Type": "application/json"}, # mutable-ok: base request type requires dict headers
|
||||
)
|
||||
|
||||
def transform_text_to_speech_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
) -> "HttpxBinaryResponseContent":
|
||||
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
||||
|
||||
try:
|
||||
payload: Final = FlowSpeechResponse.model_validate(raw_response.json())
|
||||
except ValueError as exc:
|
||||
raise FlowSpeechException(
|
||||
status_code=raw_response.status_code,
|
||||
message="FlowSpeech API returned an invalid JSON response",
|
||||
headers=raw_response.headers,
|
||||
) from exc
|
||||
|
||||
if payload.code != 0 or payload.data is None or not payload.data.audioBase64:
|
||||
raise FlowSpeechException(
|
||||
status_code=raw_response.status_code,
|
||||
message=payload.message or "FlowSpeech API response did not include audio data",
|
||||
headers=raw_response.headers,
|
||||
)
|
||||
|
||||
try:
|
||||
audio_bytes: Final = base64.b64decode(payload.data.audioBase64, validate=True)
|
||||
except (binascii.Error, ValueError) as exc:
|
||||
raise FlowSpeechException(
|
||||
status_code=raw_response.status_code,
|
||||
message="FlowSpeech API returned invalid base64 audio data",
|
||||
headers=raw_response.headers,
|
||||
) from exc
|
||||
|
||||
preserved_headers: Final = { # mutable-ok: httpx.Response requires materialized headers
|
||||
key: value
|
||||
for key, value in raw_response.headers.items()
|
||||
if key.lower() not in frozenset({"content-encoding", "content-length", "content-type", "transfer-encoding"})
|
||||
}
|
||||
response_headers: Final = { # mutable-ok: httpx.Response requires materialized headers
|
||||
**preserved_headers,
|
||||
"content-length": str(len(audio_bytes)),
|
||||
"content-type": payload.data.mimeType,
|
||||
}
|
||||
binary_response: Final = httpx.Response(
|
||||
status_code=200,
|
||||
headers=response_headers,
|
||||
content=audio_bytes,
|
||||
request=raw_response.request,
|
||||
)
|
||||
return HttpxBinaryResponseContent(binary_response)
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
model: str,
|
||||
api_base: str | None,
|
||||
litellm_params: dict[str, object], # mutable-ok: base provider interface requires dict
|
||||
) -> str:
|
||||
base_url: Final = api_base or get_secret_str("FLOWSPEECH_API_BASE") or self.TTS_BASE_URL
|
||||
normalized_base_url: Final = base_url.rstrip("/")
|
||||
if normalized_base_url.endswith(self.TTS_ENDPOINT_PATH):
|
||||
return normalized_base_url
|
||||
return f"{normalized_base_url}{self.TTS_ENDPOINT_PATH}"
|
||||
|
|
@ -8225,6 +8225,40 @@ def speech(
|
|||
client=client,
|
||||
_is_async=aspeech or False,
|
||||
)
|
||||
elif custom_llm_provider == "flowspeech":
|
||||
from litellm.llms.flowspeech.text_to_speech.transformation import (
|
||||
FlowSpeechTextToSpeechConfig,
|
||||
)
|
||||
|
||||
if text_to_speech_provider_config is None:
|
||||
text_to_speech_provider_config = ( # rebind-ok: provider config is resolved lazily for speech dispatch
|
||||
FlowSpeechTextToSpeechConfig()
|
||||
)
|
||||
|
||||
if not isinstance(text_to_speech_provider_config, FlowSpeechTextToSpeechConfig):
|
||||
raise TypeError("FlowSpeech TTS configuration has an unexpected type")
|
||||
flowspeech_config: Final = text_to_speech_provider_config
|
||||
|
||||
if api_base is not None:
|
||||
litellm_params_dict["api_base"] = api_base
|
||||
if api_key is not None:
|
||||
litellm_params_dict["api_key"] = api_key
|
||||
|
||||
voice_name: Final = voice if isinstance(voice, str) else None
|
||||
response = base_llm_http_handler.text_to_speech_handler( # rebind-ok: provider branch sets dispatch result
|
||||
model=model,
|
||||
input=input,
|
||||
voice=voice_name,
|
||||
text_to_speech_provider_config=flowspeech_config,
|
||||
text_to_speech_optional_params=optional_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
litellm_params=litellm_params_dict,
|
||||
logging_obj=logging_obj,
|
||||
timeout=timeout,
|
||||
extra_headers=extra_headers,
|
||||
client=client,
|
||||
_is_async=aspeech or False,
|
||||
)
|
||||
elif custom_llm_provider == "vertex_ai" or custom_llm_provider == "vertex_ai_beta":
|
||||
from litellm.llms.vertex_ai.text_to_speech.transformation import (
|
||||
VertexAITextToSpeechConfig,
|
||||
|
|
|
|||
|
|
@ -3783,6 +3783,7 @@ class LlmProviders(str, Enum):
|
|||
INFINITY = "infinity"
|
||||
DEEPGRAM = "deepgram"
|
||||
ELEVENLABS = "elevenlabs"
|
||||
FLOWSPEECH = "flowspeech"
|
||||
NOVITA = "novita"
|
||||
AIOHTTP_OPENAI = "aiohttp_openai"
|
||||
LANGFUSE = "langfuse"
|
||||
|
|
|
|||
|
|
@ -9309,6 +9309,12 @@ class ProviderConfigManager:
|
|||
)
|
||||
|
||||
return ElevenLabsTextToSpeechConfig()
|
||||
elif litellm.LlmProviders.FLOWSPEECH == provider:
|
||||
from litellm.llms.flowspeech.text_to_speech.transformation import (
|
||||
FlowSpeechTextToSpeechConfig,
|
||||
)
|
||||
|
||||
return FlowSpeechTextToSpeechConfig()
|
||||
elif litellm.LlmProviders.RUNWAYML == provider:
|
||||
from litellm.llms.runwayml.text_to_speech.transformation import (
|
||||
RunwayMLTextToSpeechConfig,
|
||||
|
|
|
|||
|
|
@ -17298,6 +17298,14 @@
|
|||
"/v1/audio/transcriptions"
|
||||
]
|
||||
},
|
||||
"flowspeech/flowspeech-tts": {
|
||||
"litellm_provider": "flowspeech",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://flowspeech.io/",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v3": {
|
||||
"input_cost_per_character": 0.00018,
|
||||
"litellm_provider": "elevenlabs",
|
||||
|
|
|
|||
1
tests/test_litellm/llms/flowspeech/__init__.py
Normal file
1
tests/test_litellm/llms/flowspeech/__init__.py
Normal file
|
|
@ -0,0 +1 @@
|
|||
|
||||
|
|
@ -0,0 +1,168 @@
|
|||
import base64
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.custom_httpx.http_handler import HTTPHandler
|
||||
from litellm.llms.flowspeech.text_to_speech.transformation import (
|
||||
FlowSpeechException,
|
||||
FlowSpeechTextToSpeechConfig,
|
||||
)
|
||||
|
||||
|
||||
def test_maps_voice_and_instructions_to_flowspeech_request():
|
||||
config = FlowSpeechTextToSpeechConfig()
|
||||
voice, params = config.map_openai_params(
|
||||
model="flowspeech-tts",
|
||||
optional_params={"instructions": "Speak with calm confidence"},
|
||||
voice={"voiceName": "Aoede"},
|
||||
)
|
||||
|
||||
request = config.transform_text_to_speech_request(
|
||||
model="flowspeech-tts",
|
||||
input="Hello from FlowSpeech",
|
||||
voice=voice,
|
||||
optional_params=params,
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["dict_body"] == {
|
||||
"text": "Hello from FlowSpeech",
|
||||
"originalText": "Hello from FlowSpeech",
|
||||
"speakers": [{"voiceName": "Aoede"}],
|
||||
"prompt": "Speak with calm confidence",
|
||||
}
|
||||
|
||||
|
||||
def test_uses_default_voice_and_ignores_unsupported_openai_params():
|
||||
config = FlowSpeechTextToSpeechConfig()
|
||||
voice, params = config.map_openai_params(
|
||||
model="flowspeech-tts",
|
||||
optional_params={"response_format": "mp3", "speed": 1.5},
|
||||
)
|
||||
|
||||
assert voice == "Kore"
|
||||
assert params == {}
|
||||
|
||||
|
||||
def test_validates_bearer_api_key():
|
||||
config = FlowSpeechTextToSpeechConfig()
|
||||
|
||||
headers = config.validate_environment({}, "flowspeech-tts", api_key="test-key")
|
||||
|
||||
assert headers == {
|
||||
"Authorization": "Bearer test-key",
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
|
||||
def test_requires_api_key(monkeypatch):
|
||||
config = FlowSpeechTextToSpeechConfig()
|
||||
monkeypatch.setattr(litellm, "api_key", None)
|
||||
monkeypatch.delenv("FLOWSPEECH_API_KEY", raising=False)
|
||||
|
||||
with pytest.raises(ValueError, match="FlowSpeech API key is required"):
|
||||
config.validate_environment({}, "flowspeech-tts")
|
||||
|
||||
|
||||
def test_decodes_flowspeech_audio_response():
|
||||
config = FlowSpeechTextToSpeechConfig()
|
||||
audio = b"test audio bytes"
|
||||
response = httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "application/json", "x-request-id": "request-1"},
|
||||
json={
|
||||
"code": 0,
|
||||
"data": {
|
||||
"mimeType": "audio/mpeg",
|
||||
"audioBase64": base64.b64encode(audio).decode(),
|
||||
},
|
||||
},
|
||||
request=httpx.Request("POST", "https://flowspeech.io/api/ai/text-to-speech"),
|
||||
)
|
||||
|
||||
result = config.transform_text_to_speech_response("flowspeech-tts", response, logging_obj=None)
|
||||
|
||||
assert result.content == audio
|
||||
assert result.response.headers["content-type"] == "audio/mpeg"
|
||||
assert result.response.headers["x-request-id"] == "request-1"
|
||||
|
||||
|
||||
def test_rejects_api_error_response():
|
||||
config = FlowSpeechTextToSpeechConfig()
|
||||
response = httpx.Response(
|
||||
400,
|
||||
json={"code": 4001, "message": "Quota exceeded", "data": None},
|
||||
request=httpx.Request("POST", "https://flowspeech.io/api/ai/text-to-speech"),
|
||||
)
|
||||
|
||||
with pytest.raises(FlowSpeechException, match="Quota exceeded"):
|
||||
config.transform_text_to_speech_response("flowspeech-tts", response, logging_obj=None)
|
||||
|
||||
|
||||
def test_builds_default_and_custom_urls():
|
||||
config = FlowSpeechTextToSpeechConfig()
|
||||
|
||||
assert config.get_complete_url("flowspeech-tts", None, {}) == "https://flowspeech.io/api/ai/text-to-speech"
|
||||
assert (
|
||||
config.get_complete_url(
|
||||
"flowspeech-tts",
|
||||
"https://example.com/api/ai/text-to-speech",
|
||||
{},
|
||||
)
|
||||
== "https://example.com/api/ai/text-to-speech"
|
||||
)
|
||||
|
||||
|
||||
def test_registers_flowspeech_provider_and_config():
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
model, provider, _, _ = litellm.get_llm_provider("flowspeech/flowspeech-tts")
|
||||
config = ProviderConfigManager.get_provider_text_to_speech_config(
|
||||
model=model,
|
||||
provider=litellm.LlmProviders.FLOWSPEECH,
|
||||
)
|
||||
|
||||
assert model == "flowspeech-tts"
|
||||
assert provider == "flowspeech"
|
||||
assert isinstance(config, FlowSpeechTextToSpeechConfig)
|
||||
|
||||
|
||||
def test_speech_dispatches_to_flowspeech_handler():
|
||||
audio = b"generated audio"
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
assert request.url == "https://flowspeech.io/api/ai/text-to-speech"
|
||||
assert request.headers["authorization"] == "Bearer test-key"
|
||||
assert json.loads(request.content) == {
|
||||
"text": "Hello",
|
||||
"originalText": "Hello",
|
||||
"speakers": [{"voiceName": "Aoede"}],
|
||||
"prompt": "Speak warmly",
|
||||
}
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"code": 0,
|
||||
"data": {
|
||||
"mimeType": "audio/mpeg",
|
||||
"audioBase64": base64.b64encode(audio).decode(),
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
client = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
|
||||
result = litellm.speech(
|
||||
model="flowspeech/flowspeech-tts",
|
||||
input="Hello",
|
||||
voice="Aoede",
|
||||
instructions="Speak warmly",
|
||||
api_key="test-key",
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert result.content == audio
|
||||
assert result.response.headers["content-type"] == "audio/mpeg"
|
||||
Loading…
Add table
Reference in a new issue