mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-14 04:09:09 +00:00
test(integration): cover audio endpoint contracts (#44699)
* test(integration): cover audio endpoint contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): close audio SDK clients after each test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): avoid list mutation in audio tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): keep exact upstream call check in model resolution test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover audio translations, bracket spellings, streaming and scoped alias keys Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): pin audio metadata parse sites, exact cost and speech bug pins Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): drop inline comment on audio wire assertion Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): strengthen audio wire and authorization contracts Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo <mateo@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot]
GitHub
parent
35124f3b49
commit
8a2f706ab7
5 changed files with 1222 additions and 0 deletions
104
tests/integration/authorization/test_audio_model_access.py
Normal file
104
tests/integration/authorization/test_audio_model_access.py
Normal file
|
|
@ -0,0 +1,104 @@
|
|||
import json
|
||||
import mimetypes
|
||||
from email.parser import BytesParser
|
||||
from email.policy import HTTP
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
_WAV_BYTES: Final = (
|
||||
b"RIFF\x30\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00"
|
||||
b"\x40\x1f\x00\x00\x40\x1f\x00\x00\x01\x00\x08\x00data\x08\x00\x00\x00"
|
||||
b"\x00\x01\x02\x03\x04\x05\x06\x07"
|
||||
)
|
||||
_WAV_CONTENT_TYPE: Final = mimetypes.guess_type("a.wav")[0] or "application/octet-stream"
|
||||
_SPEECH_BYTES: Final = bytes(range(256)) * 32
|
||||
|
||||
|
||||
def _transcription_model(request: Request) -> str:
|
||||
envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body
|
||||
parsed: Final = BytesParser(policy=HTTP).parsebytes(envelope)
|
||||
assert parsed.is_multipart(), request.headers["content-type"]
|
||||
fields: Final = {
|
||||
part.get_param("name", header="content-disposition"): part.get_payload(decode=True).decode()
|
||||
for part in parsed.iter_parts()
|
||||
if part.get_filename() is None
|
||||
}
|
||||
return fields["model"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ("/v1/audio/transcriptions", "/audio/transcriptions"))
|
||||
def test_audio_transcription_enforces_key_model_access(gateway: Gateway, path: str) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
assert _transcription_model(request) == "whisper-1"
|
||||
return Reply(body=json.dumps({"text": "hello world"}).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
allowed: Final = scenario.model(
|
||||
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
other: Final = scenario.model(
|
||||
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
key: Final = scenario.key(models=[allowed])
|
||||
denied: Final = gateway.request_multipart(
|
||||
path, {"model": other}, {"file": ("a.wav", _WAV_BYTES, _WAV_CONTENT_TYPE)}, key=key
|
||||
)
|
||||
assert denied.status_code == 403, denied.text
|
||||
error: Final = denied.json()["error"]
|
||||
assert error["type"] == "key_model_access_denied" and error["param"] == "model", denied.text
|
||||
assert wire.drain() == ()
|
||||
granted: Final = gateway.request_multipart(
|
||||
path, {"model": allowed}, {"file": ("a.wav", _WAV_BYTES, _WAV_CONTENT_TYPE)}, key=key
|
||||
)
|
||||
assert granted.status_code == 200, granted.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [
|
||||
("POST", "/v1/audio/transcriptions")
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ("/v1/audio/speech", "/audio/speech"))
|
||||
def test_audio_speech_enforces_key_model_access(gateway: Gateway, path: str) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/speech"
|
||||
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
|
||||
assert json.loads(request.body) == {
|
||||
"model": "gpt-4o-mini-tts",
|
||||
"input": "hello",
|
||||
"voice": "alloy",
|
||||
}
|
||||
return Reply(body=_SPEECH_BYTES, content_type="audio/wav")
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
allowed: Final = scenario.model(
|
||||
model="openai/gpt-4o-mini-tts", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
other: Final = scenario.model(
|
||||
model="openai/gpt-4o-mini-tts", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
key: Final = scenario.key(models=[allowed])
|
||||
denied: Final = gateway.request(
|
||||
"POST",
|
||||
path,
|
||||
{"model": other, "input": "hello", "voice": "alloy"},
|
||||
key=key,
|
||||
)
|
||||
assert denied.status_code == 403, denied.text
|
||||
error: Final = denied.json()["error"]
|
||||
assert error["type"] == "key_model_access_denied" and error["param"] == "model", denied.text
|
||||
assert wire.drain() == ()
|
||||
granted: Final = gateway.request(
|
||||
"POST",
|
||||
path,
|
||||
{"model": allowed, "input": "hello", "voice": "alloy"},
|
||||
key=key,
|
||||
)
|
||||
assert granted.status_code == 200, granted.text
|
||||
assert granted.content == _SPEECH_BYTES
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/speech")]
|
||||
184
tests/integration/compatibility/test_audio_speech_wire.py
Normal file
184
tests/integration/compatibility/test_audio_speech_wire.py
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from openai import OpenAI
|
||||
|
||||
_SPEECH_BYTES: Final = bytes(range(256)) * 32
|
||||
_SPEECH_REQUEST: Final = {
|
||||
"model": "gpt-4o-mini-tts",
|
||||
"input": "hello",
|
||||
"voice": "alloy",
|
||||
"response_format": "wav",
|
||||
"speed": 1.25,
|
||||
"instructions": "cheerful",
|
||||
}
|
||||
_FORMAT_MEDIA_TYPES: Final = {
|
||||
"wav": "audio/wav",
|
||||
"flac": "audio/flac",
|
||||
"pcm": "audio/pcm",
|
||||
"opus": "audio/opus",
|
||||
}
|
||||
|
||||
|
||||
def _sdk(gateway: Gateway, path: str, api_key: str) -> OpenAI:
|
||||
return OpenAI(
|
||||
base_url=str(gateway.client.base_url).rstrip("/") + path,
|
||||
api_key=api_key,
|
||||
max_retries=0,
|
||||
http_client=httpx.Client(trust_env=False),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("base_path", ("/v1", ""))
|
||||
def test_audio_speech_forwards_the_full_json_body_and_streams_the_audio(gateway: Gateway, base_path: str) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/speech"
|
||||
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
|
||||
body: Final = json.loads(request.body)
|
||||
assert body == _SPEECH_REQUEST
|
||||
assert type(body["speed"]) is float
|
||||
return Reply(body=_SPEECH_BYTES, content_type="audio/wav")
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/gpt-4o-mini-tts", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
key: Final = scenario.key(models=[model])
|
||||
with _sdk(gateway, base_path, key) as sdk:
|
||||
response: Final = sdk.audio.speech.with_raw_response.create(
|
||||
model=model,
|
||||
input="hello",
|
||||
voice="alloy",
|
||||
response_format="wav",
|
||||
speed=1.25,
|
||||
instructions="cheerful",
|
||||
)
|
||||
assert response.status_code == 200, response.read()
|
||||
assert response.content == _SPEECH_BYTES
|
||||
assert response.headers["content-type"] == "audio/wav"
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/speech")]
|
||||
|
||||
|
||||
def test_audio_speech_streams_chunked_audio_through_intact(gateway: Gateway) -> None:
|
||||
audio: Final = bytes(index % 256 for index in range(64 * 1024))
|
||||
chunks: Final = tuple(audio[start : start + 4096] for start in range(0, len(audio), 4096))
|
||||
assert len(chunks) == 16
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/speech"
|
||||
return Reply(chunks=chunks, content_type="audio/wav")
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/gpt-4o-mini-tts", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
with _sdk(gateway, "/v1", gateway.key) as sdk:
|
||||
response: Final = sdk.audio.speech.with_raw_response.create(
|
||||
model=model,
|
||||
input="hello",
|
||||
voice="alloy",
|
||||
response_format="mp3",
|
||||
)
|
||||
assert response.status_code == 200, response.read()
|
||||
assert response.content == audio
|
||||
assert response.headers["content-type"] == "audio/wav"
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/speech")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_format,media_type", tuple(_FORMAT_MEDIA_TYPES.items()))
|
||||
def test_audio_speech_derives_the_media_type_from_the_requested_format(
|
||||
gateway: Gateway, response_format: str, media_type: str
|
||||
) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/speech"
|
||||
return Reply(body=_SPEECH_BYTES, content_type="application/octet-stream")
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/gpt-4o-mini-tts", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
with _sdk(gateway, "/v1", gateway.key) as sdk:
|
||||
response: Final = sdk.audio.speech.with_raw_response.create(
|
||||
model=model,
|
||||
input="hello",
|
||||
voice="alloy",
|
||||
response_format=response_format,
|
||||
)
|
||||
assert response.status_code == 200, response.read()
|
||||
assert response.headers["content-type"] == media_type
|
||||
assert response.content == _SPEECH_BYTES
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/speech")]
|
||||
|
||||
|
||||
def test_audio_speech_forwards_stream_format_sse(gateway: Gateway) -> None:
|
||||
pytest.skip(
|
||||
"BUG: speech stream_format='sse' is dropped from the upstream JSON body, which arrives as "
|
||||
"{input, model, voice}, and the SSE reply is relabelled audio/mpeg"
|
||||
)
|
||||
sse_body: Final = (
|
||||
b'data: {"type":"speech.audio.delta","audio":"aGVsbG8="}\n\ndata: {"type":"speech.audio.done"}\n\n'
|
||||
)
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/speech"
|
||||
assert json.loads(request.body) == {
|
||||
"model": "gpt-4o-mini-tts",
|
||||
"input": "hello",
|
||||
"voice": "alloy",
|
||||
"stream_format": "sse",
|
||||
}
|
||||
return Reply(chunks=(sse_body,), content_type="text/event-stream")
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/gpt-4o-mini-tts", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
with _sdk(gateway, "/v1", gateway.key) as sdk:
|
||||
response: Final = sdk.audio.speech.with_raw_response.create(
|
||||
model=model,
|
||||
input="hello",
|
||||
voice="alloy",
|
||||
stream_format="sse",
|
||||
)
|
||||
assert response.status_code == 200, response.read()
|
||||
assert response.headers["content-type"].startswith("text/event-stream"), response.read()
|
||||
assert response.content == sse_body
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/speech")]
|
||||
|
||||
|
||||
def test_audio_speech_forwards_object_form_voice(gateway: Gateway) -> None:
|
||||
pytest.skip(
|
||||
"BUG: speech voice={'id': 'voice_123'} is rejected with 400 \"'voice' is required to be passed "
|
||||
'as a string for OpenAI TTS" and never reaches the upstream'
|
||||
)
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/speech"
|
||||
assert json.loads(request.body) == {
|
||||
"model": "gpt-4o-mini-tts",
|
||||
"input": "hello",
|
||||
"voice": {"id": "voice_123"},
|
||||
}
|
||||
return Reply(body=_SPEECH_BYTES, content_type="audio/mpeg")
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/gpt-4o-mini-tts", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/audio/speech",
|
||||
{"model": model, "input": "hello", "voice": {"id": "voice_123"}},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.content == _SPEECH_BYTES
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/speech")]
|
||||
|
|
@ -0,0 +1,673 @@
|
|||
import json
|
||||
import math
|
||||
import mimetypes
|
||||
import uuid
|
||||
from email.message import Message
|
||||
from email.parser import BytesParser
|
||||
from email.policy import HTTP
|
||||
from typing import Final, cast
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from openai import OpenAI
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
_WAV_BYTES: Final = (
|
||||
b"RIFF\x30\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00"
|
||||
b"\x40\x1f\x00\x00\x40\x1f\x00\x00\x01\x00\x08\x00data\x08\x00\x00\x00"
|
||||
b"\x00\x01\x02\x03\x04\x05\x06\x07"
|
||||
)
|
||||
_WAV_CONTENT_TYPE: Final = mimetypes.guess_type("a.wav")[0] or "application/octet-stream"
|
||||
_VERBOSE_TEXT_PARTS: Final = (
|
||||
("include[]", "logprobs"),
|
||||
("language", "en"),
|
||||
("model", "whisper-1"),
|
||||
("prompt", "hi"),
|
||||
("response_format", "verbose_json"),
|
||||
("temperature", "0.2"),
|
||||
("timestamp_granularities[]", "word"),
|
||||
("timestamp_granularities[]", "segment"),
|
||||
)
|
||||
_VERBOSE_TRANSCRIPT: Final = {
|
||||
"task": "transcribe",
|
||||
"language": "english",
|
||||
"duration": 1.5,
|
||||
"text": "hello world",
|
||||
"words": [
|
||||
{"word": "hello", "start": 0.0, "end": 0.5},
|
||||
{"word": "world", "start": 0.6, "end": 1.0},
|
||||
],
|
||||
"segments": [
|
||||
{
|
||||
"id": 0,
|
||||
"seek": 0,
|
||||
"start": 0.0,
|
||||
"end": 1.0,
|
||||
"text": "hello world",
|
||||
"tokens": [1, 2],
|
||||
"temperature": 0.2,
|
||||
"avg_logprob": -0.1,
|
||||
"compression_ratio": 1.0,
|
||||
"no_speech_prob": 0.01,
|
||||
}
|
||||
],
|
||||
"usage": {
|
||||
"type": "tokens",
|
||||
"input_tokens": 12,
|
||||
"output_tokens": 4,
|
||||
"total_tokens": 16,
|
||||
"input_token_details": {"text_tokens": 2, "audio_tokens": 10},
|
||||
},
|
||||
}
|
||||
_JSON_TRANSCRIPT: Final = {"text": "hello world", "usage": _VERBOSE_TRANSCRIPT["usage"]}
|
||||
_PLAIN_TRANSCRIPTS: Final = {
|
||||
"text": "hello world\n",
|
||||
"srt": "1\n00:00:00,000 --> 00:00:01,000\nhello world\n",
|
||||
"vtt": "WEBVTT\n\n00:00:00.000 --> 00:00:01.000\nhello world\n",
|
||||
}
|
||||
|
||||
|
||||
class _Word(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
word: str
|
||||
start: float
|
||||
end: float
|
||||
|
||||
|
||||
class _Segment(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
id: int
|
||||
seek: int
|
||||
start: float
|
||||
end: float
|
||||
text: str
|
||||
tokens: tuple[int, ...]
|
||||
temperature: float
|
||||
avg_logprob: float
|
||||
compression_ratio: float
|
||||
no_speech_prob: float
|
||||
|
||||
|
||||
class _InputTokenDetails(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
text_tokens: int
|
||||
audio_tokens: int
|
||||
|
||||
|
||||
class _Usage(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
type: str
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
total_tokens: int
|
||||
input_token_details: _InputTokenDetails
|
||||
|
||||
|
||||
class _VerboseTranscript(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
task: str
|
||||
language: str
|
||||
duration: float
|
||||
text: str
|
||||
words: tuple[_Word, ...]
|
||||
segments: tuple[_Segment, ...]
|
||||
usage: _Usage
|
||||
|
||||
|
||||
def _multipart_parts(request: Request) -> tuple[Message, ...]:
|
||||
envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body
|
||||
parsed: Final = BytesParser(policy=HTTP).parsebytes(envelope)
|
||||
assert parsed.is_multipart(), request.headers["content-type"]
|
||||
return tuple(parsed.iter_parts())
|
||||
|
||||
|
||||
def _text_parts(parts: tuple[Message, ...]) -> tuple[tuple[str, str], ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
(
|
||||
(part.get_param("name", header="content-disposition"), part.get_payload(decode=True).decode())
|
||||
for part in parts
|
||||
if part.get_filename() is None
|
||||
),
|
||||
key=lambda field: field[0],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _file_parts(parts: tuple[Message, ...]) -> tuple[tuple[str, str, str, bytes], ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
(
|
||||
(
|
||||
part.get_param("name", header="content-disposition"),
|
||||
part.get_filename(),
|
||||
part.get_content_type(),
|
||||
part.get_payload(decode=True),
|
||||
)
|
||||
for part in parts
|
||||
if part.get_filename() is not None
|
||||
),
|
||||
key=lambda field: field[0],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _sdk(gateway: Gateway, path: str, api_key: str) -> OpenAI:
|
||||
return OpenAI(
|
||||
base_url=str(gateway.client.base_url).rstrip("/") + path,
|
||||
api_key=api_key,
|
||||
max_retries=0,
|
||||
http_client=httpx.Client(trust_env=False),
|
||||
)
|
||||
|
||||
|
||||
def test_audio_transcription_verbose_json_forwards_every_form_field_and_returns_the_transcript(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
call_id: Final = f"audio-transcription-{uuid.uuid4().hex}"
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _text_parts(parts) == _VERBOSE_TEXT_PARTS
|
||||
assert _file_parts(parts) == (("file", "a.wav", _WAV_CONTENT_TYPE, wav),)
|
||||
return Reply(body=json.dumps(_VERBOSE_TRANSCRIPT).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/whisper-1",
|
||||
api_base=f"{wire.url}/v1",
|
||||
api_key="synthetic-openai-key",
|
||||
input_cost_per_token=0.000001,
|
||||
input_cost_per_audio_token=0.000003,
|
||||
output_cost_per_token=0.000002,
|
||||
)
|
||||
with _sdk(gateway, "/v1", gateway.key) as sdk:
|
||||
response: Final = sdk.audio.transcriptions.with_raw_response.create(
|
||||
model=model,
|
||||
file=("a.wav", wav, _WAV_CONTENT_TYPE),
|
||||
temperature=0.2,
|
||||
language="en",
|
||||
prompt="hi",
|
||||
response_format="verbose_json",
|
||||
timestamp_granularities=["word", "segment"],
|
||||
include=["logprobs"],
|
||||
extra_headers={"x-litellm-call-id": call_id},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert json.loads(response.content) == _VERBOSE_TRANSCRIPT, response.text
|
||||
_VerboseTranscript.model_validate_json(response.content)
|
||||
assert response.headers["x-litellm-call-id"] == call_id, response.text
|
||||
usage: Final = cast("dict[str, object]", _VERBOSE_TRANSCRIPT["usage"])
|
||||
token_details: Final = cast("dict[str, int]", usage["input_token_details"])
|
||||
assert math.isclose(
|
||||
float(response.headers["x-litellm-response-cost"]),
|
||||
token_details["audio_tokens"] * 0.000003
|
||||
+ token_details["text_tokens"] * 0.000001
|
||||
+ cast(int, usage["output_tokens"]) * 0.000002,
|
||||
rel_tol=1e-9,
|
||||
), response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/transcriptions")]
|
||||
|
||||
|
||||
def test_audio_transcription_reaches_upstream_identically_on_prefixed_and_unprefixed_routes(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _text_parts(parts) == _VERBOSE_TEXT_PARTS
|
||||
assert _file_parts(parts) == (("file", "a.wav", _WAV_CONTENT_TYPE, wav),)
|
||||
return Reply(body=json.dumps(_VERBOSE_TRANSCRIPT).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
prefixed: Final = scenario.model(
|
||||
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
unprefixed: Final = scenario.model(
|
||||
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
key: Final = scenario.key(models=[prefixed, unprefixed])
|
||||
|
||||
def transcribe(base_path: str, model: str) -> dict[str, object]:
|
||||
with _sdk(gateway, base_path, key) as sdk:
|
||||
return json.loads(
|
||||
sdk.audio.transcriptions.with_raw_response.create(
|
||||
model=model,
|
||||
file=("a.wav", wav, _WAV_CONTENT_TYPE),
|
||||
temperature=0.2,
|
||||
language="en",
|
||||
prompt="hi",
|
||||
response_format="verbose_json",
|
||||
timestamp_granularities=["word", "segment"],
|
||||
include=["logprobs"],
|
||||
).content
|
||||
)
|
||||
|
||||
bodies: Final = tuple(
|
||||
transcribe(base_path, model) for base_path, model in (("/v1", prefixed), ("", unprefixed))
|
||||
)
|
||||
assert bodies[0] == bodies[1] == _VERBOSE_TRANSCRIPT
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [
|
||||
("POST", "/v1/audio/transcriptions"),
|
||||
("POST", "/v1/audio/transcriptions"),
|
||||
]
|
||||
|
||||
|
||||
def test_audio_transcription_json_format_returns_text_and_usage(gateway: Gateway) -> None:
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _text_parts(parts) == (("model", "whisper-1"), ("response_format", "json"))
|
||||
assert _file_parts(parts) == (("file", "a.wav", _WAV_CONTENT_TYPE, wav),)
|
||||
return Reply(body=json.dumps(_JSON_TRANSCRIPT).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
with _sdk(gateway, "/v1", gateway.key) as sdk:
|
||||
response: Final = sdk.audio.transcriptions.with_raw_response.create(
|
||||
model=model,
|
||||
file=("a.wav", wav, _WAV_CONTENT_TYPE),
|
||||
response_format="json",
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert json.loads(response.content) == _JSON_TRANSCRIPT, response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/transcriptions")]
|
||||
|
||||
|
||||
def test_audio_transcription_forwards_nested_and_repeated_bracket_fields_verbatim(gateway: Gateway) -> None:
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _text_parts(parts) == (
|
||||
("chunking_strategy[prefix_padding_ms]", "300"),
|
||||
("chunking_strategy[silence_duration_ms]", "500"),
|
||||
("chunking_strategy[threshold]", "0.5"),
|
||||
("chunking_strategy[type]", "server_vad"),
|
||||
("known_speaker_names[]", "alice"),
|
||||
("known_speaker_names[]", "bob"),
|
||||
("known_speaker_references[]", "data:audio/wav;base64,AAAA"),
|
||||
("known_speaker_references[]", "data:audio/wav;base64,BBBB"),
|
||||
("model", "gpt-4o-transcribe"),
|
||||
("response_format", "json"),
|
||||
)
|
||||
assert _file_parts(parts) == (("file", "a.wav", _WAV_CONTENT_TYPE, wav),)
|
||||
return Reply(body=json.dumps(_JSON_TRANSCRIPT).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/gpt-4o-transcribe", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
known_speaker_names: Final = ("alice", "bob")
|
||||
known_speaker_references: Final = (
|
||||
"data:audio/wav;base64,AAAA",
|
||||
"data:audio/wav;base64,BBBB",
|
||||
)
|
||||
with _sdk(gateway, "/v1", gateway.key) as sdk:
|
||||
response: Final = sdk.audio.transcriptions.with_raw_response.create(
|
||||
model=model,
|
||||
file=("a.wav", wav, _WAV_CONTENT_TYPE),
|
||||
response_format="json",
|
||||
chunking_strategy={
|
||||
"type": "server_vad",
|
||||
"prefix_padding_ms": 300,
|
||||
"silence_duration_ms": 500,
|
||||
"threshold": 0.5,
|
||||
},
|
||||
known_speaker_names=known_speaker_names,
|
||||
known_speaker_references=known_speaker_references,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert json.loads(response.content) == _JSON_TRANSCRIPT, response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/transcriptions")]
|
||||
|
||||
|
||||
def test_audio_transcription_stream_returns_server_sent_events(gateway: Gateway) -> None:
|
||||
pytest.skip(
|
||||
"BUG: stream=true transcriptions reach the upstream but its SSE body comes back as application/json "
|
||||
'{"text": "data: ..."}, so OpenAI SDK stream=True callers get no transcript events'
|
||||
)
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _text_parts(parts) == (("model", "gpt-4o-transcribe"), ("stream", "true"))
|
||||
assert _file_parts(parts) == (("file", "a.wav", _WAV_CONTENT_TYPE, wav),)
|
||||
return Reply(
|
||||
chunks=(
|
||||
b'data: {"type":"transcript.text.delta","delta":"hello"}\n\n',
|
||||
b'data: {"type":"transcript.text.delta","delta":" world"}\n\n',
|
||||
b'data: {"type":"transcript.text.done","text":"hello world",'
|
||||
b'"usage":{"type":"tokens","input_tokens":12,"output_tokens":4,"total_tokens":16}}\n\n',
|
||||
),
|
||||
content_type="text/event-stream",
|
||||
)
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/gpt-4o-transcribe", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
with _sdk(gateway, "/v1", gateway.key) as sdk:
|
||||
with sdk.audio.transcriptions.with_streaming_response.create(
|
||||
model=model,
|
||||
file=("a.wav", wav, _WAV_CONTENT_TYPE),
|
||||
stream=True,
|
||||
) as raw:
|
||||
assert raw.headers["content-type"].startswith("text/event-stream")
|
||||
events: Final = tuple(
|
||||
event.to_dict()
|
||||
for event in sdk.audio.transcriptions.create(
|
||||
model=model,
|
||||
file=("a.wav", wav, _WAV_CONTENT_TYPE),
|
||||
stream=True,
|
||||
)
|
||||
)
|
||||
assert events == (
|
||||
{"type": "transcript.text.delta", "delta": "hello"},
|
||||
{"type": "transcript.text.delta", "delta": " world"},
|
||||
{
|
||||
"type": "transcript.text.done",
|
||||
"text": "hello world",
|
||||
"usage": {
|
||||
"type": "tokens",
|
||||
"input_tokens": 12,
|
||||
"output_tokens": 4,
|
||||
"total_tokens": 16,
|
||||
},
|
||||
},
|
||||
)
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [
|
||||
("POST", "/v1/audio/transcriptions"),
|
||||
("POST", "/v1/audio/transcriptions"),
|
||||
]
|
||||
|
||||
|
||||
def test_audio_transcription_applies_json_string_metadata_without_forwarding_it(gateway: Gateway) -> None:
|
||||
tag: Final = f"audio-metadata-{uuid.uuid4().hex}"
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _text_parts(parts) == (("model", "whisper-1"), ("response_format", "json"))
|
||||
assert _file_parts(parts) == (("file", "a.wav", _WAV_CONTENT_TYPE, wav),)
|
||||
return Reply(body=json.dumps(_JSON_TRANSCRIPT).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
response: Final = gateway.request_multipart(
|
||||
"/v1/audio/transcriptions",
|
||||
{
|
||||
"model": model,
|
||||
"response_format": "json",
|
||||
"metadata": json.dumps({"tags": [tag]}),
|
||||
},
|
||||
{"file": ("a.wav", wav, _WAV_CONTENT_TYPE)},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _JSON_TRANSCRIPT, response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/transcriptions")]
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT request_tags FROM "LiteLLM_SpendLogs" WHERE request_id = %s',
|
||||
(response.headers["x-litellm-call-id"],),
|
||||
),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
request_tags: Final = rows[0]["request_tags"]
|
||||
assert isinstance(request_tags, list), rows
|
||||
assert tuple(
|
||||
value for value in request_tags if isinstance(value, str) and not value.startswith("User-Agent: ")
|
||||
) == (tag,), rows
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_format", ("text", "srt", "vtt"))
|
||||
def test_audio_transcription_forwards_plain_response_format_to_upstream(gateway: Gateway, response_format: str) -> None:
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _text_parts(parts) == (("model", "whisper-1"), ("response_format", response_format))
|
||||
assert _file_parts(parts) == (("file", "a.wav", _WAV_CONTENT_TYPE, wav),)
|
||||
return Reply(body=_PLAIN_TRANSCRIPTS[response_format].encode(), content_type="text/plain")
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
with _sdk(gateway, "/v1", gateway.key) as sdk:
|
||||
response: Final = sdk.audio.transcriptions.with_raw_response.create(
|
||||
model=model,
|
||||
file=("a.wav", wav, _WAV_CONTENT_TYPE),
|
||||
response_format=response_format,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/transcriptions")]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("response_format", ("text", "srt", "vtt"))
|
||||
def test_audio_transcription_plain_response_format_returns_the_raw_transcript(
|
||||
gateway: Gateway, response_format: str
|
||||
) -> None:
|
||||
pytest.skip(
|
||||
"BUG: response_format=text/srt/vtt transcriptions return application/json "
|
||||
'{"text": ...} instead of the raw transcript, so the OpenAI SDK returns a JSON string as the transcript'
|
||||
)
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
return Reply(body=_PLAIN_TRANSCRIPTS[response_format].encode(), content_type="text/plain")
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
with _sdk(gateway, "/v1", gateway.key) as sdk:
|
||||
response: Final = sdk.audio.transcriptions.with_raw_response.create(
|
||||
model=model,
|
||||
file=("a.wav", wav, _WAV_CONTENT_TYPE),
|
||||
response_format=response_format,
|
||||
)
|
||||
assert response.headers["content-type"].startswith("text/plain"), response.text
|
||||
transcript: Final = sdk.audio.transcriptions.create(
|
||||
model=model,
|
||||
file=("a.wav", wav, _WAV_CONTENT_TYPE),
|
||||
response_format=response_format,
|
||||
)
|
||||
assert transcript == _PLAIN_TRANSCRIPTS[response_format]
|
||||
|
||||
|
||||
def test_audio_transcription_applies_json_string_litellm_metadata_without_forwarding_it(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
tag: Final = f"audio-metadata-{uuid.uuid4().hex}"
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _text_parts(parts) == (("model", "whisper-1"), ("response_format", "json"))
|
||||
assert _file_parts(parts) == (("file", "a.wav", _WAV_CONTENT_TYPE, wav),)
|
||||
return Reply(body=json.dumps(_JSON_TRANSCRIPT).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
response: Final = gateway.request_multipart(
|
||||
"/v1/audio/transcriptions",
|
||||
{
|
||||
"model": model,
|
||||
"response_format": "json",
|
||||
"litellm_metadata": json.dumps({"tags": [tag]}),
|
||||
},
|
||||
{"file": ("a.wav", wav, _WAV_CONTENT_TYPE)},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _JSON_TRANSCRIPT, response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/transcriptions")]
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT request_tags FROM "LiteLLM_SpendLogs" WHERE request_id = %s',
|
||||
(response.headers["x-litellm-call-id"],),
|
||||
),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
request_tags: Final = rows[0]["request_tags"]
|
||||
assert isinstance(request_tags, list), rows
|
||||
assert tuple(
|
||||
value for value in request_tags if isinstance(value, str) and not value.startswith("User-Agent: ")
|
||||
) == (tag,), rows
|
||||
|
||||
|
||||
@pytest.mark.parametrize("metadata_field", ("metadata", "litellm_metadata"))
|
||||
def test_audio_transcription_consumes_bracketed_form_metadata_tags(gateway: Gateway, metadata_field: str) -> None:
|
||||
pytest.skip(
|
||||
"BUG: multipart metadata[tags]/litellm_metadata[tags] on /v1/audio/transcriptions reaches the upstream "
|
||||
"as an extra text part and the tag never lands in LiteLLM_SpendLogs.request_tags"
|
||||
)
|
||||
tag: Final = f"audio-metadata-{uuid.uuid4().hex}"
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _text_parts(parts) == (("model", "whisper-1"), ("response_format", "json"))
|
||||
assert _file_parts(parts) == (("file", "a.wav", _WAV_CONTENT_TYPE, wav),)
|
||||
return Reply(body=json.dumps(_JSON_TRANSCRIPT).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
response: Final = gateway.request_multipart(
|
||||
"/v1/audio/transcriptions",
|
||||
{
|
||||
"model": model,
|
||||
"response_format": "json",
|
||||
f"{metadata_field}[tags]": tag,
|
||||
},
|
||||
{"file": ("a.wav", wav, _WAV_CONTENT_TYPE)},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _JSON_TRANSCRIPT, response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/transcriptions")]
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT request_tags FROM "LiteLLM_SpendLogs" WHERE request_id = %s',
|
||||
(response.headers["x-litellm-call-id"],),
|
||||
),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
request_tags: Final = rows[0]["request_tags"]
|
||||
assert isinstance(request_tags, list), rows
|
||||
assert tuple(
|
||||
value for value in request_tags if isinstance(value, str) and not value.startswith("User-Agent: ")
|
||||
) == (tag,), rows
|
||||
|
||||
|
||||
def test_audio_transcription_preserves_the_client_file_content_type(gateway: Gateway) -> None:
|
||||
pytest.skip(
|
||||
"BUG: the proxy discards the client's file part content type and re-derives it from the filename, "
|
||||
"so file=('recording', ..., 'audio/wav') reaches the upstream as application/octet-stream"
|
||||
)
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _file_parts(parts) == (("file", "recording", "audio/wav", wav),)
|
||||
return Reply(body=json.dumps(_JSON_TRANSCRIPT).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
response: Final = gateway.request_multipart(
|
||||
"/v1/audio/transcriptions",
|
||||
{"model": model, "response_format": "json"},
|
||||
{"file": ("recording", wav, "audio/wav")},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _JSON_TRANSCRIPT, response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/transcriptions")]
|
||||
|
||||
|
||||
def test_audio_transcription_bills_audio_tokens_at_the_input_rate_when_no_audio_rate_is_set(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
pytest.skip(
|
||||
"BUG: with only input_cost_per_token set, transcription audio tokens are billed at 0, so "
|
||||
"x-litellm-response-cost is 1e-05 instead of 2e-05 for 10 audio, 2 text and 4 output tokens"
|
||||
)
|
||||
wav: Final = _WAV_BYTES + uuid.uuid4().bytes
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/transcriptions"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _text_parts(parts) == (("model", "whisper-1"), ("response_format", "json"))
|
||||
assert _file_parts(parts) == (("file", "a.wav", _WAV_CONTENT_TYPE, wav),)
|
||||
return Reply(body=json.dumps(_JSON_TRANSCRIPT).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/whisper-1",
|
||||
api_base=f"{wire.url}/v1",
|
||||
api_key="synthetic-openai-key",
|
||||
input_cost_per_token=0.000001,
|
||||
output_cost_per_token=0.000002,
|
||||
)
|
||||
response: Final = gateway.request_multipart(
|
||||
"/v1/audio/transcriptions",
|
||||
{"model": model, "response_format": "json"},
|
||||
{"file": ("a.wav", wav, _WAV_CONTENT_TYPE)},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _JSON_TRANSCRIPT, response.text
|
||||
usage: Final = cast("dict[str, object]", _JSON_TRANSCRIPT["usage"])
|
||||
token_details: Final = cast("dict[str, int]", usage["input_token_details"])
|
||||
assert math.isclose(
|
||||
float(response.headers["x-litellm-response-cost"]),
|
||||
token_details["audio_tokens"] * 0.000001
|
||||
+ token_details["text_tokens"] * 0.000001
|
||||
+ cast(int, usage["output_tokens"]) * 0.000002,
|
||||
rel_tol=1e-9,
|
||||
), response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/transcriptions")]
|
||||
105
tests/integration/compatibility/test_audio_translations_wire.py
Normal file
105
tests/integration/compatibility/test_audio_translations_wire.py
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
import json
|
||||
import mimetypes
|
||||
from email.message import Message
|
||||
from email.parser import BytesParser
|
||||
from email.policy import HTTP
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from openai import OpenAI
|
||||
|
||||
_WAV_BYTES: Final = (
|
||||
b"RIFF\x30\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00"
|
||||
b"\x40\x1f\x00\x00\x40\x1f\x00\x00\x01\x00\x08\x00data\x08\x00\x00\x00"
|
||||
b"\x00\x01\x02\x03\x04\x05\x06\x07"
|
||||
)
|
||||
_WAV_CONTENT_TYPE: Final = mimetypes.guess_type("a.wav")[0] or "application/octet-stream"
|
||||
|
||||
|
||||
def _multipart_parts(request: Request) -> tuple[Message, ...]:
|
||||
envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body
|
||||
parsed: Final = BytesParser(policy=HTTP).parsebytes(envelope)
|
||||
assert parsed.is_multipart(), request.headers["content-type"]
|
||||
return tuple(parsed.iter_parts())
|
||||
|
||||
|
||||
def _text_parts(parts: tuple[Message, ...]) -> tuple[tuple[str, str], ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
(
|
||||
(part.get_param("name", header="content-disposition"), part.get_payload(decode=True).decode())
|
||||
for part in parts
|
||||
if part.get_filename() is None
|
||||
),
|
||||
key=lambda field: field[0],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _file_parts(parts: tuple[Message, ...]) -> tuple[tuple[str, str, str, bytes], ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
(
|
||||
(
|
||||
part.get_param("name", header="content-disposition"),
|
||||
part.get_filename(),
|
||||
part.get_content_type(),
|
||||
part.get_payload(decode=True),
|
||||
)
|
||||
for part in parts
|
||||
if part.get_filename() is not None
|
||||
),
|
||||
key=lambda field: field[0],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _sdk(gateway: Gateway, path: str, api_key: str) -> OpenAI:
|
||||
return OpenAI(
|
||||
base_url=str(gateway.client.base_url).rstrip("/") + path,
|
||||
api_key=api_key,
|
||||
max_retries=0,
|
||||
http_client=httpx.Client(trust_env=False),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("base_path", ("/v1", ""))
|
||||
def test_audio_translation_reaches_upstream_on_prefixed_and_unprefixed_routes(gateway: Gateway, base_path: str) -> None:
|
||||
pytest.skip(
|
||||
"BUG: the proxy registers no /v1/audio/translations or /audio/translations route, so OpenAI SDK "
|
||||
"translations.create gets 404 Not Found and never reaches the provider"
|
||||
)
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/audio/translations"
|
||||
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _text_parts(parts) == (
|
||||
("model", "whisper-1"),
|
||||
("prompt", "hi"),
|
||||
("response_format", "json"),
|
||||
("temperature", "0.2"),
|
||||
)
|
||||
assert _file_parts(parts) == (("file", "a.wav", _WAV_CONTENT_TYPE, _WAV_BYTES),)
|
||||
return Reply(body=json.dumps({"text": "hello world"}).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/whisper-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
key: Final = scenario.key(models=[model])
|
||||
with _sdk(gateway, base_path, key) as sdk:
|
||||
response: Final = sdk.audio.translations.with_raw_response.create(
|
||||
model=model,
|
||||
file=("a.wav", _WAV_BYTES, _WAV_CONTENT_TYPE),
|
||||
prompt="hi",
|
||||
response_format="json",
|
||||
temperature=0.2,
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert json.loads(response.content) == {"text": "hello world"}, response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/audio/translations")]
|
||||
|
|
@ -0,0 +1,156 @@
|
|||
import json
|
||||
import mimetypes
|
||||
import uuid
|
||||
from email.message import Message
|
||||
from email.parser import BytesParser
|
||||
from email.policy import HTTP
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import JsonValue
|
||||
|
||||
from tests.integration._support.client import Gateway
|
||||
from tests.integration._support.process import owned_proxy
|
||||
from tests.integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
_WAV_BYTES: Final = (
|
||||
b"RIFF\x30\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00"
|
||||
b"\x40\x1f\x00\x00\x40\x1f\x00\x00\x01\x00\x08\x00data\x08\x00\x00\x00"
|
||||
b"\x00\x01\x02\x03\x04\x05\x06\x07"
|
||||
)
|
||||
_WAV_CONTENT_TYPE: Final = mimetypes.guess_type("a.wav")[0] or "application/octet-stream"
|
||||
_JSON_TRANSCRIPT: Final = {
|
||||
"text": "hello world",
|
||||
"usage": {
|
||||
"type": "tokens",
|
||||
"input_tokens": 12,
|
||||
"output_tokens": 4,
|
||||
"total_tokens": 16,
|
||||
"input_token_details": {"text_tokens": 2, "audio_tokens": 10},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _write_config(directory: Path, upstream_url: str, stt: str, moderation: str) -> Path:
|
||||
config: Final = directory / f"audio_transcription_{uuid.uuid4().hex}.yaml"
|
||||
litellm_params: Final[dict[str, JsonValue]] = {
|
||||
"api_base": f"{upstream_url}/v1",
|
||||
"api_key": "synthetic-openai-key",
|
||||
}
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [
|
||||
{"model_name": stt, "litellm_params": {**litellm_params, "model": "openai/whisper-1"}},
|
||||
{
|
||||
"model_name": moderation,
|
||||
"litellm_params": {**litellm_params, "model": "openai/omni-moderation-latest"},
|
||||
},
|
||||
],
|
||||
"general_settings": {
|
||||
"master_key": "os.environ/LITELLM_MASTER_KEY",
|
||||
"database_url": "os.environ/DATABASE_URL",
|
||||
"store_model_in_db": True,
|
||||
"disable_spend_logs": False,
|
||||
"proxy_batch_write_at": 1,
|
||||
"proxy_batch_polling_interval": 1,
|
||||
"moderation_model": moderation,
|
||||
},
|
||||
"litellm_settings": {
|
||||
"enable_redis_auth_cache": True,
|
||||
"cache": True,
|
||||
"cache_params": {
|
||||
"type": "redis",
|
||||
"host": "os.environ/REDIS_HOST",
|
||||
"port": "os.environ/REDIS_PORT",
|
||||
},
|
||||
},
|
||||
"router_settings": {"disable_cooldowns": True},
|
||||
}
|
||||
)
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _multipart_parts(request: Request) -> tuple[Message, ...]:
|
||||
envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body
|
||||
parsed: Final = BytesParser(policy=HTTP).parsebytes(envelope)
|
||||
assert parsed.is_multipart(), request.headers["content-type"]
|
||||
return tuple(parsed.iter_parts())
|
||||
|
||||
|
||||
def _text_parts(parts: tuple[Message, ...]) -> tuple[tuple[str, str], ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
(
|
||||
(part.get_param("name", header="content-disposition"), part.get_payload(decode=True).decode())
|
||||
for part in parts
|
||||
if part.get_filename() is None
|
||||
),
|
||||
key=lambda field: field[0],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _file_parts(parts: tuple[Message, ...]) -> tuple[tuple[str, str, str, bytes], ...]:
|
||||
return tuple(
|
||||
sorted(
|
||||
(
|
||||
(
|
||||
part.get_param("name", header="content-disposition"),
|
||||
part.get_filename(),
|
||||
part.get_content_type(),
|
||||
part.get_payload(decode=True),
|
||||
)
|
||||
for part in parts
|
||||
if part.get_filename() is not None
|
||||
),
|
||||
key=lambda field: field[0],
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
_TRANSCRIPTIONS: Final = "/v1/audio/transcriptions"
|
||||
|
||||
|
||||
@pytest.mark.timeout(240)
|
||||
def test_audio_transcription_routes_to_the_requested_model_not_the_moderation_model(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
pytest.skip(
|
||||
"BUG: general_settings.moderation_model overrides the model of every /audio/transcriptions request, "
|
||||
"so the upstream receives the moderation deployment's model"
|
||||
)
|
||||
stt: Final = f"integration-stt-{uuid.uuid4().hex}"
|
||||
moderation: Final = f"integration-moderation-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
if request.target != _TRANSCRIPTIONS:
|
||||
return Reply(body=b'{"object":"list","data":[]}')
|
||||
return Reply(body=json.dumps(_JSON_TRANSCRIPT).encode())
|
||||
|
||||
with wire_server(respond) as wire:
|
||||
config: Final = _write_config(tmp_path, wire.url, stt, moderation)
|
||||
with owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config) as candidate:
|
||||
response: Final = candidate.request_multipart(
|
||||
_TRANSCRIPTIONS,
|
||||
{"model": stt, "response_format": "json"},
|
||||
{"file": ("a.wav", _WAV_BYTES, _WAV_CONTENT_TYPE)},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json() == _JSON_TRANSCRIPT, response.text
|
||||
calls: Final = wire.drain()
|
||||
assert tuple((request.method, request.target) for request in calls if request.target != "/v1/models") == (
|
||||
("POST", _TRANSCRIPTIONS),
|
||||
), response.text
|
||||
assert tuple(
|
||||
_text_parts(_multipart_parts(request))
|
||||
for request in calls
|
||||
if request.target == _TRANSCRIPTIONS
|
||||
) == ((("model", "whisper-1"), ("response_format", "json")),), response.text
|
||||
assert tuple(
|
||||
_file_parts(_multipart_parts(request))
|
||||
for request in calls
|
||||
if request.target == _TRANSCRIPTIONS
|
||||
) == ((("file", "a.wav", _WAV_CONTENT_TYPE, _WAV_BYTES),),), response.text
|
||||
Loading…
Add table
Reference in a new issue