scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-01 07:03:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 128

feat(speechify): add text-to-speech support for /v1/audio/speech

Browse source
This commit is contained in:
Lokesh Manchanda 2026-10-31 12:00:35 +05:30
parent a392a39627
commit abc8db54aa
15 changed files with 512 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/test-unit.yml vendored
View file

@ -363,6 +363,7 @@ jobs:
tests/unit/llms/serper
tests/unit/llms/snowflake
tests/unit/llms/soniox
tests/unit/llms/speechify
tests/unit/llms/stability
tests/unit/llms/strands_decider
tests/unit/llms/tavily

1
README.md
View file

@ -457,6 +457,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th
| [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | |
| [ScaleDown (`scaledown`)](https://docs.litellm.ai/docs/providers/scaledown) | ✅ | | | | | | | | | |
| [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | |
| [Speechify (`speechify`)](https://docs.litellm.ai/docs/providers/speechify) | | | | | | | ✅ | | | |
| [Strands Decider (`strands_decider`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | |
| [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | |
| [Text Completion OpenAI (`text-completion-openai`)](https://docs.litellm.ai/docs/providers/text_completion_openai) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | |

4
litellm/__init__.py
View file

@ -714,6 +714,7 @@ microsoft_365_copilot_models: Set[str] = set()
chatgpt_models: Set = set()
minimax_models: Set = set()
aws_polly_models: Set = set()
speechify_models: Set = set() # mutable-ok: filled from the price map at import, like the sibling provider sets
transcribe_models: Set = set()
gigachat_models: Set = set()
llamagate_models: Set = set()
@ -1032,6 +1033,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
minimax_models.add(key)
elif value.get("litellm_provider") == "aws_polly":
aws_polly_models.add(key)
elif value.get("litellm_provider") == "speechify":
speechify_models.add(key)
elif value.get("litellm_provider") == "transcribe":
transcribe_models.add(key)
elif value.get("litellm_provider") == "gigachat":
@ -1322,6 +1325,7 @@ def _build_models_by_provider() -> dict:
"chatgpt": chatgpt_models,
"minimax": minimax_models,
"aws_polly": aws_polly_models,
"speechify": speechify_models,
"transcribe": transcribe_models,
"gigachat": gigachat_models,
"llamagate": llamagate_models,

180
litellm/llms/speechify/text_to_speech/transformation.py Normal file
View file

@ -0,0 +1,180 @@
"""
Speechify text-to-speech via ``POST /v1/audio/speech``.
API reference: https://docs.speechify.ai/build/api-reference/v1/audio/speech?utm_source=github&utm_medium=docs&utm_campaign=litellm
"""
import base64
import binascii
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import BaseModel, ValidationError
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.text_to_speech.transformation import (
BaseTextToSpeechConfig,
TextToSpeechRequestData,
)
from litellm.secret_managers.main import get_secret_str
if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.types.llms.openai import HttpxBinaryResponseContent
class SpeechifyTextToSpeechException(BaseLLMException):
pass
class SpeechifySpeechResponse(BaseModel):
audio_data: str
audio_format: str
class SpeechifyTextToSpeechConfig(BaseTextToSpeechConfig):
TTS_BASE_URL: Final[str] = "https://api.speechify.ai"
DEFAULT_AUDIO_FORMAT: Final[str] = "mp3"
OPENAI_TO_SPEECHIFY_FORMAT: Final[MappingProxyType[str, str]] = MappingProxyType({"opus": "ogg"})
PASSTHROUGH_PARAMS: Final[tuple[str, ...]] = ("language", "output_format")
AUDIO_CONTENT_TYPES: Final[MappingProxyType[str, str]] = MappingProxyType(
{
"mp3": "audio/mpeg",
"wav": "audio/wav",
"ogg": "audio/ogg",
"aac": "audio/aac",
"pcm": "audio/pcm",
"l16": "audio/pcm",
"ulaw": "audio/basic",
}
)
DROPPED_RESPONSE_HEADERS: Final[frozenset[str]] = frozenset(
{"content-encoding", "transfer-encoding", "content-length", "content-type"}
)
def get_supported_openai_params(self, model: str) -> list: # mutable-ok: base class contract returns a plain list
return ["voice", "response_format"] # mutable-ok: base class contract returns a plain list
def map_openai_params(
self,
model: str,
optional_params: Mapping[str, object],
voice: object = None,
drop_params: bool = False,
kwargs: Mapping[str, object] | None = None,
) -> tuple[str | None, dict]: # mutable-ok: base class contract returns a plain dict
response_format: Final = optional_params.get("response_format")
audio_format: Final = (
self.OPENAI_TO_SPEECHIFY_FORMAT.get(response_format, response_format)
if isinstance(response_format, str)
else self.DEFAULT_AUDIO_FORMAT
)
provider_params: Final = tuple((key, kwargs.get(key)) for key in self.PASSTHROUGH_PARAMS) if kwargs else ()
mapped_params: Final = { # mutable-ok: base class contract returns a plain dict
key: value for key, value in (("audio_format", audio_format), *provider_params) if isinstance(value, str)
}
mapped_voice: Final = voice.strip() if isinstance(voice, str) and voice.strip() else None
return mapped_voice, mapped_params
def validate_environment(
self,
headers: Mapping[str, str],
model: str,
api_key: str | None = None,
api_base: str | None = None,
) -> dict: # mutable-ok: base class contract returns a plain dict
resolved_key: Final = api_key or get_secret_str("SPEECHIFY_API_KEY")
if resolved_key is None:
raise SpeechifyTextToSpeechException(
status_code=401,
message="Speechify API key is required. Set SPEECHIFY_API_KEY or pass api_key.",
)
return { # mutable-ok: base class contract returns a plain dict
"Speechify-Caller": "litellm",
**headers,
"Authorization": f"Bearer {resolved_key}",
"Content-Type": "application/json",
}
def get_complete_url(
self,
model: str,
api_base: str | None,
litellm_params: Mapping[str, object],
) -> str:
configured_base: Final = (api_base or self.TTS_BASE_URL).rstrip("/")
unversioned_base: Final = configured_base.removesuffix("/v1")
return f"{unversioned_base}/v1/audio/speech"
def transform_text_to_speech_request(
self,
model: str,
input: str,
voice: str | None,
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
headers: Mapping[str, str],
) -> TextToSpeechRequestData:
request_data: Final[TextToSpeechRequestData] = {
"dict_body": {
"model": model,
"input": input,
**({"voice_id": voice} if voice else {}),
**optional_params,
},
"headers": {"Content-Type": "application/json"},
}
return request_data
def _content_type(self, audio_format: str) -> str:
codec: Final = audio_format.partition(";")[0].strip().lower()
return self.AUDIO_CONTENT_TYPES.get(codec, "application/octet-stream")
def transform_text_to_speech_response(
self,
model: str,
raw_response: httpx.Response,
logging_obj: "LiteLLMLoggingObj",
) -> "HttpxBinaryResponseContent":
from litellm.types.llms.openai import HttpxBinaryResponseContent
try:
speech: Final = SpeechifySpeechResponse.model_validate_json(raw_response.content)
audio_bytes: Final = base64.b64decode(speech.audio_data, validate=True)
except (ValidationError, binascii.Error) as e:
raise SpeechifyTextToSpeechException(
status_code=500,
message=f"Unexpected Speechify speech response: {e}",
headers=raw_response.headers,
)
retained_headers: Final = tuple(
(key, value)
for key, value in raw_response.headers.items()
if key.lower() not in self.DROPPED_RESPONSE_HEADERS
)
response_headers: Final = retained_headers + (
("content-length", str(len(audio_bytes))),
("content-type", self._content_type(speech.audio_format)),
)
return HttpxBinaryResponseContent(
httpx.Response(
status_code=200,
headers=response_headers,
content=audio_bytes,
request=raw_response.request,
)
)
def get_error_class(
self,
error_message: str,
status_code: int,
headers: dict | httpx.Headers, # mutable-ok: BaseLLMException takes a plain dict or httpx.Headers
) -> BaseLLMException:
return SpeechifyTextToSpeechException(
message=error_message,
status_code=status_code,
headers=headers,
)

22
litellm/main.py
View file

@ -9288,6 +9288,28 @@ def speech(
client=client,
_is_async=aspeech or False,
)
elif custom_llm_provider == "speechify":
from litellm.llms.speechify.text_to_speech.transformation import (
SpeechifyTextToSpeechConfig,
)
if api_base is not None:
litellm_params_dict["api_base"] = api_base
response = base_llm_http_handler.text_to_speech_handler(
model=model,
input=input,
voice=voice if isinstance(voice, str) else None,
text_to_speech_provider_config=text_to_speech_provider_config or SpeechifyTextToSpeechConfig(),
text_to_speech_optional_params=optional_params,
custom_llm_provider=custom_llm_provider,
litellm_params=litellm_params_dict,
logging_obj=logging_obj,
timeout=timeout,
extra_headers=extra_headers,
client=client,
_is_async=aspeech or False,
)
elif custom_llm_provider == "aws_polly":
from litellm.llms.aws_polly.text_to_speech.transformation import (
AWSPollyTextToSpeechConfig,

36
litellm/model_prices_and_context_window_backup.json
View file

@ -81321,5 +81321,41 @@
"supports_response_schema": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
},
"speechify/simba-3.5": {
"input_cost_per_character": 2e-05,
"litellm_provider": "speechify",
"metadata": {
"calculation": "$20 per 1M characters (Starter plan list price; Pro is $16 and Scale is $12 per 1M)"
},
"mode": "audio_speech",
"source": "https://speechify.ai/pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"speechify/simba-3.2": {
"input_cost_per_character": 1e-05,
"litellm_provider": "speechify",
"metadata": {
"calculation": "$10 per 1M characters (Starter plan pay-as-you-go rate)"
},
"mode": "audio_speech",
"source": "https://speechify.ai/pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"speechify/simba-3.0": {
"input_cost_per_character": 1e-05,
"litellm_provider": "speechify",
"metadata": {
"calculation": "$10 per 1M characters (Starter plan pay-as-you-go rate)"
},
"mode": "audio_speech",
"source": "https://speechify.ai/pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
}
}

16
litellm/provider_endpoints_support_backup.json
View file

@ -3786,6 +3786,22 @@
"generateContent": false
}
},
"speechify": {
"display_name": "Speechify (`speechify`)",
"url": "https://docs.litellm.ai/docs/providers/speechify",
"endpoints": {
"chat_completions": false,
"messages": false,
"responses": false,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": true,
"moderations": false,
"batches": false,
"rerank": false
}
},
"synthetic": {
"display_name": "Synthetic (`synthetic`)",
"endpoints": {

18
litellm/proxy/public_endpoints/provider_create_fields.json
View file

@ -3705,6 +3705,24 @@
],
"default_model_placeholder": "soniox/stt-async-v5"
},
{
"provider": "Speechify",
"provider_display_name": "Speechify",
"litellm_provider": "speechify",
"credential_fields": [
{
"key": "api_key",
"label": "Speechify API Key",
"placeholder": null,
"tooltip": null,
"required": true,
"field_type": "password",
"options": null,
"default_value": null
}
],
"default_model_placeholder": "speechify/simba-3.5"
},
{
"provider": "StrandsDecider",
"provider_display_name": "Strands Decider",

1
litellm/types/utils.py
View file

@ -4694,6 +4694,7 @@ class LlmProviders(str, Enum):
BEDROCK_MANTLE = "bedrock_mantle"
GDC = "gdc"
KYMA = "kyma"
SPEECHIFY = "speechify"
# Create a set of all provider values for quick lookup

6
litellm/utils.py
View file

@ -11285,6 +11285,12 @@ class ProviderConfigManager:
)
return AWSPollyTextToSpeechConfig()
elif litellm.LlmProviders.SPEECHIFY == provider:
from litellm.llms.speechify.text_to_speech.transformation import (
SpeechifyTextToSpeechConfig,
)
return SpeechifyTextToSpeechConfig()
elif litellm.LlmProviders.EDENAI == provider:
return litellm.EdenAITextToSpeechConfig()
elif litellm.LlmProviders.DASHSCOPE == provider:

36
model_prices_and_context_window.json
View file

@ -81321,5 +81321,41 @@
"supports_response_schema": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
},
"speechify/simba-3.5": {
"input_cost_per_character": 2e-05,
"litellm_provider": "speechify",
"metadata": {
"calculation": "$20 per 1M characters (Starter plan list price; Pro is $16 and Scale is $12 per 1M)"
},
"mode": "audio_speech",
"source": "https://speechify.ai/pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"speechify/simba-3.2": {
"input_cost_per_character": 1e-05,
"litellm_provider": "speechify",
"metadata": {
"calculation": "$10 per 1M characters (Starter plan pay-as-you-go rate)"
},
"mode": "audio_speech",
"source": "https://speechify.ai/pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"speechify/simba-3.0": {
"input_cost_per_character": 1e-05,
"litellm_provider": "speechify",
"metadata": {
"calculation": "$10 per 1M characters (Starter plan pay-as-you-go rate)"
},
"mode": "audio_speech",
"source": "https://speechify.ai/pricing",
"supported_endpoints": [
"/v1/audio/speech"
]
}
}

16
provider_endpoints_support.json
View file

@ -3786,6 +3786,22 @@
"generateContent": false
}
},
"speechify": {
"display_name": "Speechify (`speechify`)",
"url": "https://docs.litellm.ai/docs/providers/speechify",
"endpoints": {
"chat_completions": false,
"messages": false,
"responses": false,
"embeddings": false,
"image_generations": false,
"audio_transcriptions": false,
"audio_speech": true,
"moderations": false,
"batches": false,
"rerank": false
}
},
"synthetic": {
"display_name": "Synthetic (`synthetic`)",
"endpoints": {

0
tests/unit/llms/speechify/__init__.py Normal file
View file

0
tests/unit/llms/speechify/text_to_speech/__init__.py Normal file
View file

175
tests/unit/llms/speechify/text_to_speech/test_speechify_text_to_speech_transformation.py Normal file
View file

@ -0,0 +1,175 @@
import base64
import json
from typing import Final
from unittest.mock import MagicMock
import httpx
import pytest
import litellm
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
from litellm.llms.speechify.text_to_speech.transformation import (
SpeechifyTextToSpeechConfig,
SpeechifyTextToSpeechException,
)
SPEECH_URL: Final = "https://api.speechify.ai/v1/audio/speech"
AUDIO_BYTES: Final = b"ID3-fake-mp3-bytes"
def _speechify_reply(audio_format: str = "mp3") -> dict[str, object]:
return {
"audio_data": base64.b64encode(AUDIO_BYTES).decode(),
"audio_format": audio_format,
"billable_characters_count": 5,
"speech_marks": {},
}
def _client_recording(
requests: list[httpx.Request], reply: httpx.Response
) -> AsyncHTTPHandler: # mutable-ok: collects requests the mock transport sees
def handle(request: httpx.Request) -> httpx.Response:
requests.append(request)
return reply
return AsyncHTTPHandler(transport=httpx.MockTransport(handle))
async def test_speech_sends_speechify_request_and_returns_decoded_audio():
requests: Final[list[httpx.Request]] = [] # mutable-ok: filled by the mock transport
client: Final = _client_recording(requests, httpx.Response(200, json=_speechify_reply()))
response: Final = await litellm.aspeech(
model="speechify/simba-3.5",
input="hello",
voice="geffen_32",
api_key="sk-speechify-test",
client=client,
)
assert response.content == AUDIO_BYTES
assert response.response.headers["content-type"] == "audio/mpeg"
assert response.response.headers["content-length"] == str(len(AUDIO_BYTES))
(request,) = requests
assert str(request.url) == SPEECH_URL
assert request.headers["authorization"] == "Bearer sk-speechify-test"
assert json.loads(request.content) == {
"model": "simba-3.5",
"input": "hello",
"voice_id": "geffen_32",
"audio_format": "mp3",
}
async def test_speech_maps_speechify_errors_by_their_status():
requests: Final[list[httpx.Request]] = [] # mutable-ok: filled by the mock transport
error_body: Final = {"error": {"code": "voice_not_found", "message": "voice not found"}}
client: Final = _client_recording(requests, httpx.Response(404, json=error_body))
with pytest.raises(litellm.NotFoundError, match="voice_not_found"):
await litellm.aspeech(
model="speechify/simba-3.5",
input="hello",
voice="not_a_voice",
api_key="sk-speechify-test",
client=client,
)
@pytest.mark.parametrize(
("response_format", "audio_format"),
[(None, "mp3"), ("mp3", "mp3"), ("wav", "wav"), ("pcm", "pcm"), ("aac", "aac"), ("opus", "ogg")],
)
def test_map_openai_params_maps_response_format_to_audio_format(response_format: str | None, audio_format: str):
optional_params: Final = {} if response_format is None else {"response_format": response_format}
voice, params = SpeechifyTextToSpeechConfig().map_openai_params(
model="simba-3.5",
optional_params=optional_params,
voice=" geffen_32 ",
)
assert voice == "geffen_32"
assert params == {"audio_format": audio_format}
def test_map_openai_params_drops_unsupported_openai_params_and_keeps_speechify_ones():
_, params = SpeechifyTextToSpeechConfig().map_openai_params(
model="simba-3.0",
optional_params={"response_format": "mp3", "speed": 1.5, "instructions": "sound cheerful"},
voice="geffen_32",
kwargs={"language": "fr-FR", "output_format": "mp3_24000_128", "unrelated": "value"},
)
assert params == {"audio_format": "mp3", "language": "fr-FR", "output_format": "mp3_24000_128"}
@pytest.mark.parametrize(
"api_base",
[None, "https://api.speechify.ai", "https://api.speechify.ai/", "https://api.speechify.ai/v1"],
)
def test_get_complete_url_always_targets_the_speech_route(api_base: str | None):
url: Final = SpeechifyTextToSpeechConfig().get_complete_url(model="simba-3.5", api_base=api_base, litellm_params={})
assert url == SPEECH_URL
def test_validate_environment_reads_key_from_env(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("SPEECHIFY_API_KEY", "sk-from-env")
headers: Final = SpeechifyTextToSpeechConfig().validate_environment(headers={}, model="simba-3.5")
assert headers["Authorization"] == "Bearer sk-from-env"
def test_validate_environment_requires_key(monkeypatch: pytest.MonkeyPatch):
monkeypatch.delenv("SPEECHIFY_API_KEY", raising=False)
with pytest.raises(SpeechifyTextToSpeechException, match="SPEECHIFY_API_KEY"):
SpeechifyTextToSpeechConfig().validate_environment(headers={}, model="simba-3.5")
@pytest.mark.parametrize(
("returned_format", "content_type"),
[
("wav", "audio/wav"),
("ogg", "audio/ogg"),
("L16;rate=24000;channels=1", "audio/pcm"),
("new-codec", "application/octet-stream"),
],
)
def test_transform_response_content_type_follows_the_returned_format(returned_format: str, content_type: str):
raw_response: Final = httpx.Response(
200,
json=_speechify_reply(audio_format=returned_format),
headers={"speechify-request-id": "req-123"},
request=httpx.Request("POST", SPEECH_URL),
)
result: Final = SpeechifyTextToSpeechConfig().transform_text_to_speech_response(
model="simba-3.5",
raw_response=raw_response,
logging_obj=MagicMock(),
)
assert result.content == AUDIO_BYTES
assert result.response.headers["content-type"] == content_type
assert result.response.headers["speechify-request-id"] == "req-123"
@pytest.mark.parametrize(
"body",
[
b"not json",
json.dumps({"audio_format": "mp3"}).encode(),
json.dumps(_speechify_reply() | {"audio_data": "QUJD!"}).encode(),
],
)
def test_transform_response_rejects_malformed_bodies(body: bytes):
raw_response: Final = httpx.Response(200, content=body, request=httpx.Request("POST", SPEECH_URL))
with pytest.raises(SpeechifyTextToSpeechException, match="Unexpected Speechify speech response"):
SpeechifyTextToSpeechConfig().transform_text_to_speech_response(
model="simba-3.5",
raw_response=raw_response,
logging_obj=MagicMock(),
)
def test_price_map_lists_simba_3_5_at_the_per_character_rate():
entry: Final = litellm.model_cost["speechify/simba-3.5"]
assert entry["litellm_provider"] == "speechify"
assert entry["mode"] == "audio_speech"
assert entry["input_cost_per_character"] == 2e-05
assert "speechify/simba-3.5" in litellm.speechify_models