mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-15 04:19:08 +00:00
dfe4df8c8a
* test: delete unconditionally skipped legacy tests * test: move whole-unit legacy test files into tests/unit * test: keep moved legacy tests free of import-time global state * test: keep the module-level invocation scan pointed at tests/local_testing * ci: drop the agent_testing CircleCI job emptied by the move * test: fix moved-test isolation and router coverage * test: add Tinyfish search package marker * test: isolate moved tests from logger state leaks * test: isolate Helicone logging fixture state * test: isolate Vertex pass-through credentials between moved tests * test: cancel S3 periodic flush tasks started by moved tests * ci: restore CircleCI assistant test selection after move * test: prevent Datadog datetime import shadowing * ci: drop the litellm_assistants_api_testing CircleCI job emptied by the move * test: deduplicate imports in rebased unit tests * test: remove duplicate passthrough router patch import * test: remove moved legacy source files after rebase * test: align moved tests with rebased main * test: carry main's legacy-file edits into moved destinations * test: make the moved cost map fallback tests assert the fetch and the backup The four fallback cases only checked the result was non-empty, so they still passed with integrity validation disabled. They now inject a mock client, assert one fetch happened, and assert the result is exactly the local backup with the fallback reason recorded. --------- Co-authored-by: yuneng <yuneng@berri.ai>
538 lines
16 KiB
Python
538 lines
16 KiB
Python
# What is this?
|
|
## unit tests for openai tts endpoint
|
|
|
|
import os
|
|
|
|
from dotenv import load_dotenv
|
|
|
|
load_dotenv()
|
|
|
|
from pathlib import Path
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
|
|
|
import pytest
|
|
|
|
import litellm
|
|
|
|
|
|
async def _run_audio_speech_litellm(sync_mode, model, api_base, api_key):
|
|
litellm.turn_on_debug()
|
|
speech_file_path = Path(__file__).parent / "speech.mp3"
|
|
|
|
if sync_mode:
|
|
response = litellm.speech(
|
|
model=model,
|
|
voice="alloy",
|
|
input="the quick brown fox jumped over the lazy dogs",
|
|
api_base=api_base,
|
|
api_key=api_key,
|
|
organization=None,
|
|
project=None,
|
|
max_retries=1,
|
|
timeout=600,
|
|
client=None,
|
|
optional_params={},
|
|
)
|
|
|
|
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
|
|
|
assert isinstance(response, HttpxBinaryResponseContent)
|
|
else:
|
|
response = await litellm.aspeech(
|
|
model=model,
|
|
voice="alloy",
|
|
input="the quick brown fox jumped over the lazy dogs",
|
|
api_base=api_base,
|
|
api_key=api_key,
|
|
organization=None,
|
|
project=None,
|
|
max_retries=1,
|
|
timeout=600,
|
|
client=None,
|
|
optional_params={},
|
|
)
|
|
|
|
from litellm.llms.openai.openai import HttpxBinaryResponseContent
|
|
|
|
assert isinstance(response, HttpxBinaryResponseContent)
|
|
|
|
|
|
@pytest.mark.parametrize("sync_mode", [True, False])
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.flaky(retries=3, delay=1)
|
|
async def test_audio_speech_litellm_azure(sync_mode):
|
|
await _run_audio_speech_litellm(
|
|
sync_mode=sync_mode,
|
|
model="azure/tts",
|
|
api_base=os.getenv("AZURE_TTS_API_BASE"),
|
|
api_key=os.getenv("AZURE_TTS_API_KEY"),
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("sync_mode", [True, False])
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.flaky(retries=3, delay=1)
|
|
async def test_audio_speech_litellm_openai(sync_mode):
|
|
await _run_audio_speech_litellm(
|
|
sync_mode=sync_mode,
|
|
model="openai/tts-1",
|
|
api_base=None,
|
|
api_key=os.getenv("OPENAI_API_KEY"),
|
|
)
|
|
|
|
|
|
|
|
|
|
@pytest.mark.flaky(retries=6, delay=2)
|
|
@pytest.mark.asyncio
|
|
async def test_speech_litellm_vertex_async():
|
|
# Mock the response
|
|
mock_response = AsyncMock()
|
|
|
|
def return_val():
|
|
return {
|
|
"audioContent": "dGVzdCByZXNwb25zZQ==",
|
|
}
|
|
|
|
mock_response.json = return_val
|
|
mock_response.status_code = 200
|
|
|
|
# Set up the mock for asynchronous calls
|
|
with patch(
|
|
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
|
|
new_callable=AsyncMock,
|
|
) as mock_async_post:
|
|
mock_async_post.return_value = mock_response
|
|
model = "vertex_ai/test"
|
|
|
|
try:
|
|
response = await litellm.aspeech(
|
|
model=model,
|
|
input="async hello what llm guardrail do you have",
|
|
)
|
|
except litellm.APIConnectionError as e:
|
|
if "Your default credentials were not found" in str(e):
|
|
pytest.skip("skipping test, credentials not found")
|
|
|
|
# Assert asynchronous call
|
|
mock_async_post.assert_called_once()
|
|
_, kwargs = mock_async_post.call_args
|
|
print("call args", kwargs)
|
|
|
|
assert kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize"
|
|
|
|
assert "x-goog-user-project" in kwargs["headers"]
|
|
assert kwargs["headers"]["Authorization"] is not None
|
|
|
|
assert kwargs["json"] == {
|
|
"input": {"text": "async hello what llm guardrail do you have"},
|
|
"voice": {"languageCode": "en-US", "name": "en-US-Studio-O"},
|
|
"audioConfig": {"audioEncoding": "LINEAR16", "speakingRate": "1"},
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_speech_litellm_vertex_async_with_voice():
|
|
# Mock the response
|
|
mock_response = AsyncMock()
|
|
|
|
def return_val():
|
|
return {
|
|
"audioContent": "dGVzdCByZXNwb25zZQ==",
|
|
}
|
|
|
|
mock_response.json = return_val
|
|
mock_response.status_code = 200
|
|
|
|
# Set up the mock for asynchronous calls
|
|
with patch(
|
|
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
|
|
new_callable=AsyncMock,
|
|
) as mock_async_post:
|
|
mock_async_post.return_value = mock_response
|
|
model = "vertex_ai/test"
|
|
|
|
try:
|
|
response = await litellm.aspeech(
|
|
model=model,
|
|
input="async hello what llm guardrail do you have",
|
|
voice={
|
|
"languageCode": "en-UK",
|
|
"name": "en-UK-Studio-O",
|
|
},
|
|
audioConfig={
|
|
"audioEncoding": "LINEAR22",
|
|
"speakingRate": "10",
|
|
},
|
|
)
|
|
except litellm.APIConnectionError as e:
|
|
if "Your default credentials were not found" in str(e):
|
|
pytest.skip("skipping test, credentials not found")
|
|
|
|
# Assert asynchronous call
|
|
mock_async_post.assert_called_once()
|
|
_, kwargs = mock_async_post.call_args
|
|
print("call args", kwargs)
|
|
|
|
assert kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize"
|
|
|
|
assert "x-goog-user-project" in kwargs["headers"]
|
|
assert kwargs["headers"]["Authorization"] is not None
|
|
|
|
assert kwargs["json"] == {
|
|
"input": {"text": "async hello what llm guardrail do you have"},
|
|
"voice": {"languageCode": "en-UK", "name": "en-UK-Studio-O"},
|
|
"audioConfig": {"audioEncoding": "LINEAR22", "speakingRate": "10"},
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_speech_litellm_vertex_async_with_voice_ssml():
|
|
# Mock the response
|
|
mock_response = AsyncMock()
|
|
|
|
def return_val():
|
|
return {
|
|
"audioContent": "dGVzdCByZXNwb25zZQ==",
|
|
}
|
|
|
|
mock_response.json = return_val
|
|
mock_response.status_code = 200
|
|
|
|
ssml = """
|
|
<speak>
|
|
<p>Hello, world!</p>
|
|
<p>This is a test of the <break strength="medium" /> text-to-speech API.</p>
|
|
</speak>
|
|
"""
|
|
|
|
# Set up the mock for asynchronous calls
|
|
with patch(
|
|
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post",
|
|
new_callable=AsyncMock,
|
|
) as mock_async_post:
|
|
mock_async_post.return_value = mock_response
|
|
model = "vertex_ai/test"
|
|
|
|
try:
|
|
response = await litellm.aspeech(
|
|
input=ssml,
|
|
model=model,
|
|
voice={
|
|
"languageCode": "en-UK",
|
|
"name": "en-UK-Studio-O",
|
|
},
|
|
audioConfig={
|
|
"audioEncoding": "LINEAR22",
|
|
"speakingRate": "10",
|
|
},
|
|
)
|
|
except litellm.APIConnectionError as e:
|
|
if "Your default credentials were not found" in str(e):
|
|
pytest.skip("skipping test, credentials not found")
|
|
|
|
# Assert asynchronous call
|
|
mock_async_post.assert_called_once()
|
|
_, kwargs = mock_async_post.call_args
|
|
print("call args", kwargs)
|
|
|
|
assert kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize"
|
|
|
|
assert "x-goog-user-project" in kwargs["headers"]
|
|
assert kwargs["headers"]["Authorization"] is not None
|
|
|
|
assert kwargs["json"] == {
|
|
"input": {"ssml": ssml},
|
|
"voice": {"languageCode": "en-UK", "name": "en-UK-Studio-O"},
|
|
"audioConfig": {"audioEncoding": "LINEAR22", "speakingRate": "10"},
|
|
}
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.flaky(retries=3, delay=1)
|
|
async def test_azure_ava_tts_async():
|
|
"""
|
|
Test Azure AVA (Cognitive Services) Text-to-Speech with real API request.
|
|
"""
|
|
litellm.turn_on_debug()
|
|
api_key = os.getenv("AZURE_TTS_API_KEY")
|
|
api_base = os.getenv("AZURE_TTS_API_BASE")
|
|
|
|
speech_file_path = Path(__file__).parent / "azure_speech.mp3"
|
|
|
|
try:
|
|
response = await litellm.aspeech(
|
|
model="azure/tts",
|
|
voice="alloy",
|
|
input="Hello, this is a test of Azure text to speech",
|
|
api_base=api_base,
|
|
api_key=api_key,
|
|
response_format="mp3",
|
|
speed=1.0,
|
|
)
|
|
|
|
# Assert the response is HttpxBinaryResponseContent
|
|
from litellm.types.llms.openai import HttpxBinaryResponseContent
|
|
|
|
assert isinstance(response, HttpxBinaryResponseContent)
|
|
|
|
# Get the binary content
|
|
binary_content = response.content
|
|
assert len(binary_content) > 0
|
|
|
|
# MP3 files start with these magic bytes
|
|
# ID3 tag or MPEG sync word
|
|
assert (
|
|
binary_content[:3] == b"ID3"
|
|
or binary_content[:2] == b"\xff\xfb"
|
|
or binary_content[:2] == b"\xff\xf3"
|
|
)
|
|
|
|
# Write to file
|
|
response.stream_to_file(speech_file_path)
|
|
|
|
# Verify file was created and has content
|
|
assert speech_file_path.exists()
|
|
assert speech_file_path.stat().st_size > 0
|
|
|
|
print(f"Azure TTS audio saved to: {speech_file_path}")
|
|
|
|
except Exception as e:
|
|
pytest.fail(f"Test failed with exception: {str(e)}")
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_azure_ava_tts_with_custom_voice():
|
|
"""
|
|
Test that when using a custom Azure voice (en-US-AndrewNeural),
|
|
the SSML request body contains the selected voice.
|
|
"""
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
|
|
# Mock response
|
|
mock_response_content = b"fake_audio_data"
|
|
mock_httpx_response = MagicMock(spec=httpx.Response)
|
|
mock_httpx_response.content = mock_response_content
|
|
mock_httpx_response.status_code = 200
|
|
mock_httpx_response.headers = {"content-type": "audio/mpeg"}
|
|
|
|
with patch(
|
|
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post"
|
|
) as mock_post:
|
|
mock_post.return_value = mock_httpx_response
|
|
|
|
response = await litellm.aspeech(
|
|
model="azure/speech/azure-tts",
|
|
voice="en-US-AndrewNeural",
|
|
input="Hello, this is a test",
|
|
api_base="https://eastus.tts.speech.microsoft.com",
|
|
api_key="fake-key",
|
|
response_format="mp3",
|
|
)
|
|
|
|
# Verify the mock was called
|
|
assert mock_post.called
|
|
|
|
# Get the call arguments
|
|
call_args = mock_post.call_args
|
|
ssml_body = call_args.kwargs.get("data")
|
|
|
|
# Verify the SSML contains the custom voice
|
|
assert ssml_body is not None
|
|
assert "en-US-AndrewNeural" in ssml_body
|
|
assert "Hello, this is a test" in ssml_body
|
|
assert "<speak" in ssml_body
|
|
assert "<voice" in ssml_body
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_azure_ava_tts_fable_voice_mapping():
|
|
"""
|
|
Test that when using OpenAI voice 'fable',
|
|
it gets mapped to Azure voice 'en-GB-RyanNeural' in the SSML.
|
|
"""
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
|
|
# Mock response
|
|
mock_response_content = b"fake_audio_data"
|
|
mock_httpx_response = MagicMock(spec=httpx.Response)
|
|
mock_httpx_response.content = mock_response_content
|
|
mock_httpx_response.status_code = 200
|
|
mock_httpx_response.headers = {"content-type": "audio/mpeg"}
|
|
|
|
with patch(
|
|
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post"
|
|
) as mock_post:
|
|
mock_post.return_value = mock_httpx_response
|
|
|
|
response = await litellm.aspeech(
|
|
model="azure/speech/azure-tts",
|
|
voice="fable",
|
|
input="Testing voice mapping",
|
|
api_base="https://eastus.tts.speech.microsoft.com",
|
|
api_key="fake-key",
|
|
response_format="mp3",
|
|
)
|
|
|
|
# Verify the mock was called
|
|
assert mock_post.called
|
|
|
|
# Get the call arguments
|
|
call_args = mock_post.call_args
|
|
ssml_body = call_args.kwargs.get("data")
|
|
|
|
# Verify the SSML contains the mapped voice (en-GB-RyanNeural, not 'fable')
|
|
assert ssml_body is not None
|
|
assert "en-GB-RyanNeural" in ssml_body
|
|
assert "fable" not in ssml_body.lower()
|
|
assert "Testing voice mapping" in ssml_body
|
|
assert "<speak" in ssml_body
|
|
assert "<voice" in ssml_body
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_aws_polly_tts_with_native_voice():
|
|
"""
|
|
Test AWS Polly TTS with a native Polly voice (Joanna).
|
|
Verifies the request is formatted correctly for the Polly API.
|
|
"""
|
|
import json
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
|
|
# Mock response - Polly returns audio bytes directly
|
|
mock_response_content = b"fake_audio_data"
|
|
mock_httpx_response = MagicMock(spec=httpx.Response)
|
|
mock_httpx_response.content = mock_response_content
|
|
mock_httpx_response.status_code = 200
|
|
mock_httpx_response.headers = {"content-type": "audio/mpeg"}
|
|
|
|
with patch(
|
|
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post"
|
|
) as mock_post:
|
|
mock_post.return_value = mock_httpx_response
|
|
|
|
response = await litellm.aspeech(
|
|
model="aws_polly/neural",
|
|
voice="Joanna",
|
|
input="Hello, this is a test of AWS Polly",
|
|
aws_region_name="us-east-1",
|
|
)
|
|
|
|
# Verify the mock was called
|
|
assert mock_post.called
|
|
|
|
# Get the call arguments - AWS Polly uses data= with JSON string (for SigV4 signing)
|
|
call_args = mock_post.call_args
|
|
request_data = call_args.kwargs.get("data")
|
|
|
|
# Parse the JSON body
|
|
assert request_data is not None
|
|
request_body = json.loads(request_data)
|
|
|
|
# Verify the request body is formatted correctly for Polly
|
|
assert request_body["VoiceId"] == "Joanna"
|
|
assert request_body["Text"] == "Hello, this is a test of AWS Polly"
|
|
assert request_body["OutputFormat"] == "mp3"
|
|
assert request_body["Engine"] == "neural"
|
|
assert request_body.get("TextType", "text") == "text"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_aws_polly_tts_with_openai_voice_mapping():
|
|
"""
|
|
Test AWS Polly TTS with OpenAI voice mapping (alloy -> Joanna).
|
|
Verifies that OpenAI voices are correctly mapped to Polly voices.
|
|
"""
|
|
import json
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
|
|
mock_response_content = b"fake_audio_data"
|
|
mock_httpx_response = MagicMock(spec=httpx.Response)
|
|
mock_httpx_response.content = mock_response_content
|
|
mock_httpx_response.status_code = 200
|
|
mock_httpx_response.headers = {"content-type": "audio/mpeg"}
|
|
|
|
with patch(
|
|
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post"
|
|
) as mock_post:
|
|
mock_post.return_value = mock_httpx_response
|
|
|
|
response = await litellm.aspeech(
|
|
model="aws_polly/neural",
|
|
voice="alloy",
|
|
input="Testing OpenAI voice mapping",
|
|
aws_region_name="us-east-1",
|
|
)
|
|
|
|
assert mock_post.called
|
|
|
|
call_args = mock_post.call_args
|
|
request_data = call_args.kwargs.get("data")
|
|
|
|
# Parse the JSON body
|
|
assert request_data is not None
|
|
request_body = json.loads(request_data)
|
|
|
|
# Verify alloy was mapped to Joanna
|
|
assert request_body["VoiceId"] == "Joanna"
|
|
assert request_body["Text"] == "Testing OpenAI voice mapping"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_aws_polly_tts_with_ssml():
|
|
"""
|
|
Test AWS Polly TTS with SSML input.
|
|
Verifies that SSML is detected and TextType is set correctly.
|
|
"""
|
|
import json
|
|
from unittest.mock import patch
|
|
|
|
import httpx
|
|
|
|
mock_response_content = b"fake_audio_data"
|
|
mock_httpx_response = MagicMock(spec=httpx.Response)
|
|
mock_httpx_response.content = mock_response_content
|
|
mock_httpx_response.status_code = 200
|
|
mock_httpx_response.headers = {"content-type": "audio/mpeg"}
|
|
|
|
ssml_input = '<speak>Hello, <break time="500ms"/> this is SSML.</speak>'
|
|
|
|
with patch(
|
|
"litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post"
|
|
) as mock_post:
|
|
mock_post.return_value = mock_httpx_response
|
|
|
|
response = await litellm.aspeech(
|
|
model="aws_polly/neural",
|
|
voice="Joanna",
|
|
input=ssml_input,
|
|
aws_region_name="us-east-1",
|
|
)
|
|
|
|
assert mock_post.called
|
|
|
|
call_args = mock_post.call_args
|
|
request_data = call_args.kwargs.get("data")
|
|
|
|
# Parse the JSON body
|
|
assert request_data is not None
|
|
request_body = json.loads(request_data)
|
|
|
|
# Verify SSML is detected and TextType is set to ssml
|
|
assert request_body["Text"] == ssml_input
|
|
assert request_body["TextType"] == "ssml"
|
|
assert request_body["VoiceId"] == "Joanna"
|
|
|
|
|