mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(audio): keep transcription's positional arguments and redact streamed error details
- Move the new languages, keywords and stream options to the end of transcription's parameters, so a prompt or format passed by position keeps its meaning - Redact internal hostnames, paths and tracebacks from a streamed transcription's mid-stream error event, as chat streams already do - Move the CA-bundle check for the realtime HTTP routes, which serves real TLS on loopback, from the unit suite to the SDK integration suite
This commit is contained in:
parent
a4e4760338
commit
b33eb142d8
6 changed files with 121 additions and 115 deletions
|
|
@ -8622,13 +8622,10 @@ def transcription(
|
|||
file: FileTypes,
|
||||
## OPTIONAL OPENAI PARAMS ##
|
||||
language: str | None = None,
|
||||
languages: Sequence[str] | None = None,
|
||||
keywords: Sequence[str] | None = None,
|
||||
prompt: str | None = None,
|
||||
response_format: Literal["json", "text", "srt", "verbose_json", "vtt"] | None = None,
|
||||
timestamp_granularities: list[Literal["word", "segment"]] | None = None,
|
||||
temperature: int | None = None, # openai defaults this to 0
|
||||
stream: bool | None = None,
|
||||
## LITELLM PARAMS ##
|
||||
user: str | None = None,
|
||||
timeout=600, # default to 10 minutes
|
||||
|
|
@ -8638,6 +8635,9 @@ def transcription(
|
|||
max_retries: int | None = None,
|
||||
custom_llm_provider=None,
|
||||
base_url: str | None = None,
|
||||
languages: Sequence[str] | None = None,
|
||||
keywords: Sequence[str] | None = None,
|
||||
stream: bool | None = None,
|
||||
**kwargs,
|
||||
) -> (
|
||||
TranscriptionResponse
|
||||
|
|
|
|||
|
|
@ -324,6 +324,7 @@ from litellm._internal_context import service_target, with_service_target
|
|||
from litellm._logging import (
|
||||
STARTUP_OUTPUT_SETTINGS,
|
||||
print_startup_message,
|
||||
redact_internal_details_from_client_message,
|
||||
redact_string,
|
||||
verbose_proxy_logger,
|
||||
verbose_router_logger,
|
||||
|
|
@ -10607,7 +10608,9 @@ def _transcription_sse_error_frame(client_error: Exception) -> str:
|
|||
"error": sse_error_payload(client_error)[1]
|
||||
if isinstance(client_error, HTTPException)
|
||||
else ProxyException(
|
||||
message=getattr(client_error, "message", str(client_error)),
|
||||
message=redact_internal_details_from_client_message(
|
||||
str(getattr(client_error, "message", client_error))
|
||||
),
|
||||
type=openai_error_type(client_error, status_code),
|
||||
param=openai_error_param(client_error),
|
||||
code=status_code,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
import json
|
||||
from collections.abc import Awaitable
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.tls import server_context, write_self_signed_cert
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.realtime_api import main as realtime_main
|
||||
|
||||
UPSTREAM_PATHS: Final = MappingProxyType(
|
||||
{
|
||||
"client_secrets": "/v1/realtime/client_secrets",
|
||||
"translation_client_secrets": "/v1/realtime/translations/client_secrets",
|
||||
"calls": "/v1/realtime/calls",
|
||||
"translation_calls": "/v1/realtime/translations/calls",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _logging(route: str) -> Logging:
|
||||
return Logging(
|
||||
model="gpt-realtime",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="acreate_realtime_client_secret",
|
||||
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
|
||||
litellm_call_id=f"test-realtime-{route}",
|
||||
function_id=f"test-realtime-{route}",
|
||||
)
|
||||
|
||||
|
||||
def _route_call(route: str, api_base: str) -> Awaitable[httpx.Response]:
|
||||
match route:
|
||||
case "client_secrets":
|
||||
return realtime_main.acreate_realtime_client_secret.__wrapped__(
|
||||
model="gpt-realtime",
|
||||
custom_llm_provider="openai",
|
||||
api_key="sk-test",
|
||||
api_base=api_base,
|
||||
litellm_logging_obj=_logging(route),
|
||||
)
|
||||
case "translation_client_secrets":
|
||||
return realtime_main.acreate_realtime_translation_client_secret.__wrapped__(
|
||||
model="gpt-realtime-translate",
|
||||
custom_llm_provider="openai",
|
||||
api_key="sk-test",
|
||||
api_base=api_base,
|
||||
litellm_logging_obj=_logging(route),
|
||||
)
|
||||
case "calls":
|
||||
return realtime_main.arealtime_calls.__wrapped__(
|
||||
openai_ephemeral_key="ek_test",
|
||||
sdp_body=b"v=0",
|
||||
model="gpt-realtime",
|
||||
custom_llm_provider="openai",
|
||||
api_base=api_base,
|
||||
litellm_logging_obj=_logging(route),
|
||||
)
|
||||
case _:
|
||||
return realtime_main.arealtime_translation_calls.__wrapped__(
|
||||
openai_ephemeral_key="ek_test",
|
||||
sdp_body=b"v=0",
|
||||
model="gpt-realtime-translate",
|
||||
custom_llm_provider="openai",
|
||||
api_base=api_base,
|
||||
litellm_logging_obj=_logging(route),
|
||||
)
|
||||
|
||||
|
||||
def _ephemeral_key(_: Request) -> Reply:
|
||||
return Reply(body=json.dumps({"value": "ek_test"}).encode())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("route", tuple(UPSTREAM_PATHS))
|
||||
async def test_openai_realtime_http_routes_trust_the_ca_bundle_set_in_litellm_ssl_verify(
|
||||
route: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
certificate, key = write_self_signed_cert(tmp_path)
|
||||
monkeypatch.setattr(litellm, "ssl_verify", str(certificate))
|
||||
|
||||
with wire_server(_ephemeral_key, tls=server_context(certificate, key)) as upstream:
|
||||
response: Final = await _route_call(route, upstream.url)
|
||||
targets: Final = tuple(request.target for request in upstream.drain())
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"value": "ek_test"}
|
||||
assert targets == (UPSTREAM_PATHS[route],)
|
||||
|
|
@ -930,7 +930,7 @@ class _FailsAfterTwoDeltas(httpx.AsyncByteStream):
|
|||
async def __aiter__(self) -> AsyncIterator[bytes]:
|
||||
yield b'data: {"type":"transcript.text.delta","delta":"hello "}\n\n'
|
||||
yield b'data: {"type":"transcript.text.delta","delta":"world"}\n\n'
|
||||
raise httpx.ReadError("upstream connection dropped")
|
||||
raise httpx.ReadError("upstream connection dropped at /srv/app/stt_stream.py via stt.internal.corp")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("patched_transcription_stream")
|
||||
|
|
@ -976,6 +976,8 @@ def test_streamed_transcription_error_mid_stream_ends_with_an_error_event_and_ru
|
|||
assert len(frames) == 3
|
||||
assert data_lines[-1] == "[DONE]"
|
||||
assert "upstream connection dropped" in frames[2]["error"]["message"]
|
||||
assert "/srv/app/stt_stream.py" not in frames[2]["error"]["message"]
|
||||
assert "stt.internal.corp" not in frames[2]["error"]["message"]
|
||||
assert set(frames[2]["error"]) == {"message", "type", "param", "code"}
|
||||
assert (frames[2]["error"]["type"], frames[2]["error"]["param"], frames[2]["error"]["code"]) == (
|
||||
"internal_server_error",
|
||||
|
|
|
|||
|
|
@ -1,13 +1,9 @@
|
|||
import asyncio
|
||||
import functools
|
||||
import ipaddress
|
||||
import json
|
||||
import ssl
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Iterator, Mapping
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from types import TracebackType
|
||||
from typing import ClassVar, Final, Literal, Protocol
|
||||
|
|
@ -17,10 +13,6 @@ import httpx
|
|||
import openai
|
||||
import pytest
|
||||
import respx
|
||||
from cryptography import x509
|
||||
from cryptography.hazmat.primitives import hashes, serialization
|
||||
from cryptography.hazmat.primitives.asymmetric import ec
|
||||
from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID
|
||||
from openai import AsyncOpenAI
|
||||
from websockets.exceptions import ConnectionClosed, ConnectionClosedError
|
||||
from websockets.frames import Close
|
||||
|
|
@ -2006,106 +1998,6 @@ _OPENAI_REALTIME_HTTP_ROUTE_PATHS: Final = {
|
|||
}
|
||||
|
||||
|
||||
def _validity(builder: x509.CertificateBuilder) -> x509.CertificateBuilder:
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
return (
|
||||
builder.serial_number(x509.random_serial_number())
|
||||
.not_valid_before(now - timedelta(minutes=5))
|
||||
.not_valid_after(now + timedelta(hours=1))
|
||||
)
|
||||
|
||||
|
||||
def _write_private_ca_and_loopback_certificate(directory: Path) -> tuple[Path, Path, Path]:
|
||||
ca_key: Final = ec.generate_private_key(ec.SECP256R1())
|
||||
server_key: Final = ec.generate_private_key(ec.SECP256R1())
|
||||
ca_name: Final = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "litellm realtime test CA")])
|
||||
ca_certificate: Final = (
|
||||
_validity(x509.CertificateBuilder())
|
||||
.subject_name(ca_name)
|
||||
.issuer_name(ca_name)
|
||||
.public_key(ca_key.public_key())
|
||||
.add_extension(x509.BasicConstraints(ca=True, path_length=None), critical=True)
|
||||
.add_extension(
|
||||
x509.KeyUsage(
|
||||
digital_signature=True,
|
||||
content_commitment=False,
|
||||
key_encipherment=False,
|
||||
data_encipherment=False,
|
||||
key_agreement=False,
|
||||
key_cert_sign=True,
|
||||
crl_sign=True,
|
||||
encipher_only=False,
|
||||
decipher_only=False,
|
||||
),
|
||||
critical=True,
|
||||
)
|
||||
.add_extension(x509.SubjectKeyIdentifier.from_public_key(ca_key.public_key()), critical=False)
|
||||
.sign(ca_key, hashes.SHA256())
|
||||
)
|
||||
server_certificate: Final = (
|
||||
_validity(x509.CertificateBuilder())
|
||||
.subject_name(x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "127.0.0.1")]))
|
||||
.issuer_name(ca_name)
|
||||
.public_key(server_key.public_key())
|
||||
.add_extension(x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), critical=False)
|
||||
.add_extension(x509.ExtendedKeyUsage([ExtendedKeyUsageOID.SERVER_AUTH]), critical=False)
|
||||
.add_extension(x509.AuthorityKeyIdentifier.from_issuer_public_key(ca_key.public_key()), critical=False)
|
||||
.sign(ca_key, hashes.SHA256())
|
||||
)
|
||||
ca_path: Final = directory / "ca.pem"
|
||||
certificate_path: Final = directory / "server.pem"
|
||||
key_path: Final = directory / "server.key"
|
||||
ca_path.write_bytes(ca_certificate.public_bytes(serialization.Encoding.PEM))
|
||||
certificate_path.write_bytes(server_certificate.public_bytes(serialization.Encoding.PEM))
|
||||
key_path.write_bytes(
|
||||
server_key.private_bytes(
|
||||
serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption()
|
||||
)
|
||||
)
|
||||
return ca_path, certificate_path, key_path
|
||||
|
||||
|
||||
class _EchoPathHandler(BaseHTTPRequestHandler):
|
||||
def do_POST(self) -> None:
|
||||
self.rfile.read(int(self.headers.get("Content-Length", "0")))
|
||||
body: Final = json.dumps({"value": "ek_test", "path": self.path}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def private_ca_upstream(tmp_path: Path) -> Iterator[tuple[str, Path]]:
|
||||
ca_path, certificate_path, key_path = _write_private_ca_and_loopback_certificate(tmp_path)
|
||||
tls: Final = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
tls.load_cert_chain(certificate_path, key_path)
|
||||
server: Final = ThreadingHTTPServer(("127.0.0.1", 0), _EchoPathHandler)
|
||||
server.socket = tls.wrap_socket(server.socket, server_side=True)
|
||||
thread: Final = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
yield f"https://127.0.0.1:{server.server_address[1]}", ca_path
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("route", tuple(_OPENAI_REALTIME_HTTP_ROUTE_PATHS))
|
||||
async def test_openai_realtime_http_routes_trust_the_ca_bundle_set_in_litellm_ssl_verify(
|
||||
route: str, private_ca_upstream: tuple[str, Path], monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
api_base, ca_path = private_ca_upstream
|
||||
monkeypatch.setattr(litellm, "ssl_verify", str(ca_path))
|
||||
|
||||
response: Final = await _realtime_http_route_call(
|
||||
route, _realtime_http_logging(route), custom_llm_provider="openai", api_base=api_base
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json() == {"value": "ek_test", "path": _OPENAI_REALTIME_HTTP_ROUTE_PATHS[route]}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("route", tuple(_OPENAI_REALTIME_HTTP_ROUTE_PATHS))
|
||||
async def test_openai_realtime_http_routes_surface_the_upstream_error_body_verbatim(route: str) -> None:
|
||||
|
|
|
|||
|
|
@ -6127,6 +6127,20 @@ async def test_groq_atranscription_honors_base_url_alias(respx_mock: respx.MockR
|
|||
assert response.text == "hello"
|
||||
|
||||
|
||||
def test_transcription_keeps_the_prompt_passed_by_position(respx_mock: respx.MockRouter):
|
||||
route: Final = respx_mock.post(f"{GROQ_INTERNAL_BASE}/audio/transcriptions").mock(
|
||||
return_value=httpx.Response(200, json={"text": "hello"})
|
||||
)
|
||||
|
||||
litellm.transcription(
|
||||
"groq/whisper-large-v3", GROQ_WAV_FILE, "en", "Four score", base_url=GROQ_INTERNAL_BASE, api_key="fake-key"
|
||||
)
|
||||
|
||||
sent: Final = route.calls.last.request.content
|
||||
assert b'name="language"\r\n\r\nen\r\n' in sent
|
||||
assert b'name="prompt"\r\n\r\nFour score\r\n' in sent
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"provider_body,expected_upload_measurements",
|
||||
[({"text": "hello", "duration": 2.5}, 0), ({"text": "hello"}, 1)],
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue