fix: protect FlowSpeech credentials and deadline boundaries

This commit is contained in:
heng.yin 2026-10-26 21:14:19 +08:00
parent 337b6791db
commit f248535545
6 changed files with 431 additions and 18 deletions

View file

@ -232,9 +232,12 @@ class FlowSpeechTTS:
response = yield from self.exchange("GET", f"{self.url}/{job_id}")
def completed_response(self, job: FlowSpeechJob, response: httpx.Response) -> HttpxBinaryResponseContent:
self.remaining()
result: Final = self.audio_response(response)
self.remaining()
if self.on_completed is not None:
self.on_completed(job)
self.remaining()
return result
def audio_response(self, response: httpx.Response) -> HttpxBinaryResponseContent:
@ -280,15 +283,19 @@ class FlowSpeechTTS:
return maximum
def read(self, client: httpx.Client, request: httpx.Request) -> httpx.Response:
self.remaining()
response: Final = client.send(request, stream=True, auth=None, follow_redirects=False)
try:
self.remaining()
maximum: Final = self.limit(request, response)
with io.BytesIO() as content:
self.remaining()
for chunk in response.iter_bytes():
self.remaining()
if content.tell() + len(chunk) > maximum:
raise self.error(502, "FlowSpeech response exceeds size limit")
content.write(chunk)
self.remaining()
self.remaining()
return httpx.Response(
response.status_code, headers=response.headers, content=content.getvalue(), request=request
@ -297,15 +304,19 @@ class FlowSpeechTTS:
response.close()
async def aread(self, client: httpx.AsyncClient, request: httpx.Request) -> httpx.Response:
self.remaining()
response: Final = await client.send(request, stream=True, auth=None, follow_redirects=False)
try:
self.remaining()
maximum: Final = self.limit(request, response)
with io.BytesIO() as content:
self.remaining()
async for chunk in response.aiter_bytes():
self.remaining()
if content.tell() + len(chunk) > maximum:
raise self.error(502, "FlowSpeech response exceeds size limit")
content.write(chunk)
self.remaining()
self.remaining()
return httpx.Response(
response.status_code, headers=response.headers, content=content.getvalue(), request=request

View file

@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, Final, Literal
from uuid import UUID, uuid4
import httpx
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, field_validator
from pydantic import BaseModel, ConfigDict, Field, StrictStr, TypeAdapter, ValidationError, field_validator
import litellm
from litellm.llms.base_llm.chat.transformation import BaseLLMException
@ -101,6 +101,13 @@ def validate_url(value: str) -> httpx.URL:
return url
def credential_headers(value: object) -> Mapping[str, str] | None:
try:
return TypeAdapter(Mapping[StrictStr, StrictStr]).validate_python(value)
except ValidationError:
return None
class FlowSpeechTextToSpeechConfig(BaseTextToSpeechConfig):
TTS_BASE_URL = "https://flowspeech.io"
TTS_ENDPOINT_PATH = "/api/v1/tts/jobs"
@ -147,15 +154,27 @@ class FlowSpeechTextToSpeechConfig(BaseTextToSpeechConfig):
raise ValueError("FlowSpeech voice must be a nonempty voice name")
def validate_environment(
self, headers: Mapping[str, str], model: str, api_key: str | None = None, api_base: str | None = None
self, headers: object, model: str, api_key: object = None, api_base: str | None = None
) -> dict[str, str]: # mutable-ok: base provider interface
key: Final = api_key if api_key is not None else (litellm.api_key or get_secret_str("FLOWSPEECH_API_KEY"))
if not key or not key.isascii() or any(char.isspace() or ord(char) < 33 or ord(char) == 127 for char in key):
if (
not isinstance(key, str)
or not key
or not key.isascii()
or any(char.isspace() or ord(char) < 33 or ord(char) == 127 for char in key)
):
raise ValueError("FlowSpeech API key is required and must be a nonempty token. Set FLOWSPEECH_API_KEY.")
if any(name.lower() in ("cookie", "proxy-authorization", "host", "content-length") for name in headers):
checked_headers: Final = credential_headers(headers)
if checked_headers is None:
raise ValueError("FlowSpeech headers must be a mapping of string names to string values")
if any(name.lower() in ("cookie", "proxy-authorization", "host", "content-length") for name in checked_headers):
raise ValueError("FlowSpeech does not accept cookie, proxy credential, host or content-length overrides")
return {
**{name: value for name, value in headers.items() if name.lower() not in ("authorization", "content-type")},
**{
name: value
for name, value in checked_headers.items()
if name.lower() not in ("authorization", "content-type")
},
"Authorization": f"Bearer {key}",
"Content-Type": "application/json",
}
@ -231,10 +250,11 @@ class FlowSpeechTextToSpeechConfig(BaseTextToSpeechConfig):
def get_complete_url(self, model: str, api_base: str | None, litellm_params: Mapping[str, object]) -> str:
base: Final = api_base if api_base is not None else (get_secret_str("FLOWSPEECH_API_BASE") or self.TTS_BASE_URL)
url: Final = validate_url(base)
if url.query:
if url.query or b"?" in url.raw_path:
raise ValueError("FlowSpeech api_base must not contain a query")
path: Final = url.path.rstrip("/")
if path.endswith("/api/ai/text-to-speech"):
if url.path.rstrip("/").endswith("/api/ai/text-to-speech"):
raise ValueError("FlowSpeech api_base must not use the legacy text-to-speech endpoint")
endpoint: Final = path if path.endswith(self.TTS_ENDPOINT_PATH) else path + self.TTS_ENDPOINT_PATH
return str(url.copy_with(path=endpoint))
path: Final = url.raw_path.rstrip(b"/")
suffix: Final = self.TTS_ENDPOINT_PATH.encode("ascii")
endpoint: Final = path if path.endswith(suffix) else path + suffix
return str(url.copy_with(raw_path=endpoint))

View file

@ -36,7 +36,7 @@ if TYPE_CHECKING:
import dotenv
import httpx
import openai
from pydantic import BaseModel, StrictStr, TypeAdapter, ValidationError
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import assert_never, overload
import litellm
@ -8894,11 +8894,9 @@ def speech(
flowspeech_params: Final = TypeAdapter(Mapping[str, object]).validate_python(litellm_params_dict)
flowspeech_headers: Final = flowspeech_config.validate_environment(
TypeAdapter(Mapping[StrictStr, StrictStr]).validate_python(
extra_headers or headers or MappingProxyType({})
),
TypeAdapter(object).validate_python(extra_headers or headers or MappingProxyType({})),
model,
TypeAdapter[str | None](str | None).validate_python(flowspeech_params.get("api_key"), strict=True),
flowspeech_params.get("api_key"),
api_base,
)
flowspeech_request: Final = flowspeech_config.transform_text_to_speech_request(

View file

@ -0,0 +1,213 @@
import asyncio
import io
import ipaddress
import json
import ssl
import threading
import time
import wave
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from http.server import BaseHTTPRequestHandler, HTTPServer
import httpx
import pytest
from cryptography import x509
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import ec
from cryptography.x509.oid import NameOID
import litellm
from litellm.llms.flowspeech.text_to_speech.transformation import FlowSpeechException
REQUEST_ID = "b698824b-de05-486d-8833-05d5c4522a39"
JOB_ID = "11111111-2222-4333-8444-555555555555"
BUDGET = 0.3
def wav_bytes():
buffer = io.BytesIO()
with wave.open(buffer, "wb") as audio:
audio.setparams((1, 2, 24000, 0, "NONE", "not compressed"))
audio.writeframes(b"\x00\x00" * 24)
return buffer.getvalue()
def tls_contexts(directory):
key = ec.generate_private_key(ec.SECP256R1())
name = x509.Name([x509.NameAttribute(NameOID.COMMON_NAME, "localhost")])
certificate = (
x509.CertificateBuilder()
.subject_name(name)
.issuer_name(name)
.public_key(key.public_key())
.serial_number(1)
.not_valid_before(datetime(2020, 1, 1, tzinfo=timezone.utc))
.not_valid_after(datetime(2100, 1, 1, tzinfo=timezone.utc))
.add_extension(x509.SubjectAlternativeName([x509.IPAddress(ipaddress.ip_address("127.0.0.1"))]), False)
.sign(key, hashes.SHA256())
)
cert_file = directory / "loopback.pem"
key_file = directory / "loopback.key"
cert_file.write_bytes(certificate.public_bytes(serialization.Encoding.PEM))
key_file.write_bytes(
key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.PKCS8, serialization.NoEncryption())
)
server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
server_context.load_cert_chain(cert_file, key_file)
return server_context, ssl.create_default_context(cafile=str(cert_file))
@pytest.fixture
def slow_speech_peer(tmp_path, scenario):
requests = []
body = wav_bytes()
server_context, client_context = tls_contexts(tmp_path)
class Handler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
def setup(self):
super().setup()
self.connection.settimeout(2)
def log_message(self, format, *args):
return
def reply(self, content):
self.send_response(200)
self.send_header("Content-Length", str(len(content)))
self.end_headers()
self.wfile.write(content)
self.wfile.flush()
def do_POST(self):
request_body = self.rfile.read(int(self.headers["Content-Length"]))
requests.append((self.command, self.path, dict(self.headers), json.loads(request_body)))
self.reply(
json.dumps(
{
"id": JOB_ID,
"requestId": self.headers["Idempotency-Key"],
"status": "success",
"usage": {"billableCharacters": 5, "reservedCredits": 0, "chargedCredits": 5},
"result": {
"url": f"{base_url}/audio.wav",
"mimeType": "audio/wav",
"sampleRate": 24000,
"channels": 1,
"bitsPerSample": 16,
},
}
).encode()
)
def do_GET(self):
requests.append((self.command, self.path, dict(self.headers), None))
if self.path == "/health":
self.reply(b"ok")
return
try:
if scenario == "slow_headers_body":
time.sleep(0.2)
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.flush()
time.sleep(0.2)
self.wfile.write(body)
elif scenario == "slow_header_progress":
self.wfile.write(b"HTTP/1.1 200 OK\r\nX-Progress: ")
for _ in range(10):
self.wfile.write(b"x")
self.wfile.flush()
time.sleep(0.05)
self.wfile.write(f"\r\nContent-Length: {len(body)}\r\n\r\n".encode() + body)
else:
self.send_response(200)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
for offset in range(0, len(body), 8):
time.sleep(0.05)
self.wfile.write(body[offset : offset + 8])
self.wfile.flush()
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError, ssl.SSLEOFError):
self.close_connection = True
server = HTTPServer(("127.0.0.1", 0), Handler)
server.socket = server_context.wrap_socket(server.socket, server_side=True)
base_url = f"https://127.0.0.1:{server.server_port}"
worker = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.02})
worker.start()
try:
yield base_url, client_context, requests
finally:
server.shutdown()
server.server_close()
worker.join(timeout=3)
assert not worker.is_alive()
@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["sync", "worker", "async"])
@pytest.mark.parametrize("scenario", ["slow_headers_body", "slow_body_progress", "slow_header_progress"])
async def test_cooperative_deadline_rejects_late_audio_without_retry_or_client_closure(slow_speech_peer, mode):
base_url, client_context, requests = slow_speech_peer
responses = []
completed = []
def log_event(details):
if details.get("log_event_type") == "post_api_call":
completed.append(details)
async def record_response(response):
responses.append(response)
kwargs = {
"model": "flowspeech/flowspeech-tts",
"input": "Hello",
"api_key": "synthetic-loopback-only",
"api_base": base_url,
"request_id": REQUEST_ID,
"timeout": BUDGET,
"max_retries": 2,
"logger_fn": log_event,
}
options = {
"verify": client_context,
"trust_env": False,
"limits": httpx.Limits(max_connections=1, max_keepalive_connections=1),
}
if mode == "async":
async with httpx.AsyncClient(**options, event_hooks={"response": [record_response]}) as client:
with pytest.raises(litellm.Timeout, match="total deadline") as error:
await asyncio.wait_for(litellm.aspeech(**kwargs, client=client), timeout=5)
assert not client.is_closed
assert responses and all(response.is_closed for response in responses)
assert (await client.get(f"{base_url}/health", timeout=3)).content == b"ok"
else:
with httpx.Client(**options, event_hooks={"response": [responses.append]}) as client:
if mode == "worker":
with ThreadPoolExecutor(max_workers=1) as pool:
result = pool.submit(litellm.speech, **kwargs, client=client)
with pytest.raises(FlowSpeechException, match="total deadline") as error:
result.result(timeout=5)
else:
with pytest.raises(FlowSpeechException, match="total deadline") as error:
litellm.speech(**kwargs, client=client)
assert not client.is_closed
assert responses and all(response.is_closed for response in responses)
assert client.get(f"{base_url}/health", timeout=3).content == b"ok"
assert REQUEST_ID in str(error.value)
assert JOB_ID in str(error.value)
assert [(method, path) for method, path, _, _ in requests] == [
("POST", "/api/v1/tts/jobs"),
("GET", "/audio.wav"),
("GET", "/health"),
]
assert httpx.Headers(requests[0][2])["Idempotency-Key"] == REQUEST_ID
assert requests[0][3] == {"input": {"mode": "single", "text": "Hello", "voiceName": "Kore"}}
assert "authorization" not in httpx.Headers(requests[1][2])
assert completed == []

View file

@ -11,7 +11,7 @@ import pytest
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.llms.flowspeech.text_to_speech.handler import FlowSpeechTTS
from litellm.llms.flowspeech.text_to_speech.transformation import FlowSpeechException
from litellm.llms.flowspeech.text_to_speech.transformation import FlowSpeechException, FlowSpeechTextToSpeechConfig
from litellm.types.utils import LlmProviders
REQUEST_ID = "b698824b-de05-486d-8833-05d5c4522a39"
@ -81,9 +81,9 @@ class Clock:
await asyncio.sleep(0)
def handler(clock, timeout=120, **kwargs):
def handler(clock, timeout=120, url=URL, **kwargs):
return FlowSpeechTTS(
URL,
url,
{
"Authorization": "Bearer test-key",
"Idempotency-Key": REQUEST_ID,
@ -154,6 +154,29 @@ def test_all_pending_states_and_uncredentialed_download(invoke):
assert requests[-1].extensions["timeout"]["read"] == 100
@pytest.mark.parametrize("prefix", ["/p%2Fq", "/p%2fq", "/p%252Fq", "/p%3Fq", "/p%23q"])
def test_encoded_gateway_prefix_survives_submission_and_polling(invoke, prefix):
requests = []
endpoint = f"{prefix}/api/v1/tts/jobs"
url = FlowSpeechTextToSpeechConfig().get_complete_url("flowspeech-tts", f"https://gateway.example{prefix}/", {})
def respond(request):
requests.append(request)
if request.method == "POST":
return httpx.Response(202, json=job("queued"))
if request.url.host == "gateway.example":
return httpx.Response(200, json=job())
return httpx.Response(200, content=wav_bytes())
assert invoke(handler(Clock(), url=url), respond).content == wav_bytes()
assert [(request.method, request.url.raw_path) for request in requests] == [
("POST", endpoint.encode()),
("GET", f"{endpoint}/{JOB_ID}".encode()),
("GET", b"/recording.wav"),
]
assert requests[0].headers["Idempotency-Key"] == REQUEST_ID
@pytest.mark.parametrize("failure", ["network", "server", "rate_limit"])
def test_create_retry_preserves_uuid_and_body(invoke, failure):
clock = Clock()
@ -427,6 +450,73 @@ def test_deadline_during_request(invoke):
invoke(handler(clock, timeout=10), respond)
@pytest.mark.parametrize("stage", ["submission", "download"])
@pytest.mark.parametrize("header_delay,body_delay,expected_reads", [(0.125, 0, 0), (0.075, 0.075, 1)])
def test_expired_stream_is_closed_without_more_work(invoke, stage, header_delay, body_delay, expected_reads):
clock = Clock()
completed = []
requests = []
class Stream(httpx.SyncByteStream, httpx.AsyncByteStream):
def __init__(self):
self.reads = 0
self.closed = False
def __iter__(self):
self.reads += 1
clock.now += body_delay
yield wav_bytes() if stage == "download" else json.dumps(job()).encode()
pytest.fail("must not advance a stream after its budget expires")
async def __aiter__(self):
for chunk in self:
yield chunk
def close(self):
self.closed = True
async def aclose(self):
self.close()
stream = Stream()
def respond(request):
requests.append(request)
if stage == "download" and request.method == "POST":
return httpx.Response(202, json=job())
clock.now += header_delay
return httpx.Response(200, stream=stream)
with pytest.raises(FlowSpeechException, match="total deadline") as error:
invoke(handler(clock, timeout=0.1, on_completed=completed.append), respond)
assert error.value.status_code == 408
assert REQUEST_ID in str(error.value)
assert f"job_id={JOB_ID if stage == 'download' else 'unknown'}" in str(error.value)
assert stream.reads == expected_reads
assert stream.closed
assert [request.method for request in requests] == (["POST", "GET"] if stage == "download" else ["POST"])
assert clock.sleeps == []
assert completed == []
def test_deadline_after_wav_validation_prevents_success_callback(invoke, monkeypatch):
clock = Clock()
completed = []
content = wav_bytes()
open_wav = wave.open
def delayed_open(*args, **kwargs):
clock.now = 11
return open_wav(*args, **kwargs)
monkeypatch.setattr(wave, "open", delayed_open)
responses = iter([httpx.Response(202, json=job()), httpx.Response(200, content=content)])
with pytest.raises(FlowSpeechException, match="total deadline"):
invoke(handler(clock, timeout=10, on_completed=completed.append), lambda request: next(responses))
assert completed == []
assert clock.sleeps == []
@pytest.mark.parametrize("timeout", [0, -1, float("inf"), float("nan"), httpx.Timeout(None)])
def test_invalid_timeout(timeout):
with pytest.raises(ValueError, match="timeout"):

View file

@ -1,5 +1,7 @@
import io
import json
import logging
import traceback
import wave
from uuid import UUID
@ -7,11 +9,27 @@ import httpx
import pytest
import litellm
from litellm.integrations.custom_logger import CustomLogger
from litellm.llms.custom_httpx.http_handler import HTTPHandler
from litellm.llms.flowspeech.text_to_speech.transformation import FlowSpeechTextToSpeechConfig
REQUEST_ID = "b698824b-de05-486d-8833-05d5c4522a39"
JOB_ID = "11111111-2222-4333-8444-555555555555"
SECRET_SENTINEL = "credential-value-must-not-appear-672911"
class FailureRecorder(CustomLogger):
def __init__(self):
super().__init__()
self.errors = []
def log_failure_event(self, kwargs, response_obj, start_time, end_time):
self.errors.append(
(str(kwargs["exception"]), kwargs["traceback_exception"], kwargs["standard_logging_object"]["error_str"])
)
async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
self.log_failure_event(kwargs, response_obj, start_time, end_time)
def wav_bytes():
@ -170,6 +188,7 @@ def test_complete_url(base):
("https://u:p@example.com", "FlowSpeech requires an HTTPS URL without credentials or fragment"),
("file:///tmp/test", "FlowSpeech requires an HTTPS URL without credentials or fragment"),
("https://example.com/?key=x", "FlowSpeech api_base must not contain a query"),
("https://example.com/p?", "FlowSpeech api_base must not contain a query"),
("https://example.com/#x", "FlowSpeech requires an HTTPS URL without credentials or fragment"),
(
"https://example.com/api/ai/text-to-speech",
@ -196,6 +215,68 @@ def test_gateway_base_preserves_prefix_without_duplicating_endpoint(base, expect
assert FlowSpeechTextToSpeechConfig().get_complete_url("flowspeech-tts", base, {}) == expected
@pytest.mark.parametrize("prefix", ["/p%2Fq", "/p%2fq", "/p%252Fq", "/p%3Fq", "/p%23q"])
@pytest.mark.parametrize("suffix", ["/", "/api/v1/tts/jobs", "/api/v1/tts/jobs/"])
def test_gateway_prefix_preserves_encoded_path(prefix, suffix):
url = FlowSpeechTextToSpeechConfig().get_complete_url("flowspeech-tts", f"https://gateway.example{prefix}{suffix}", {})
assert httpx.URL(url).raw_path == f"{prefix}/api/v1/tts/jobs".encode("ascii")
assert httpx.URL(url).query == b""
assert httpx.URL(url).fragment == ""
@pytest.mark.parametrize(
"headers",
[SECRET_SENTINEL, {SECRET_SENTINEL.encode(): "invalid"}, {"Authorization": {"value": SECRET_SENTINEL}}],
)
def test_invalid_headers_raise_no_input_bearing_exception_chain(headers):
with pytest.raises(ValueError, match="FlowSpeech headers must be a mapping") as error:
FlowSpeechTextToSpeechConfig().validate_environment(headers, "flowspeech-tts", "test-key")
assert error.value.__context__ is None
assert SECRET_SENTINEL not in "".join(traceback.format_exception(error.value))
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize(
"invalid,message",
[
({"api_key": SECRET_SENTINEL.encode()}, "FlowSpeech API key is required"),
({"api_key": {"value": SECRET_SENTINEL}}, "FlowSpeech API key is required"),
({"headers": {"Authorization": SECRET_SENTINEL.encode()}}, "FlowSpeech headers must be a mapping"),
({"headers": {"Authorization": {"value": SECRET_SENTINEL}}}, "FlowSpeech headers must be a mapping"),
({"extra_headers": {"Authorization": SECRET_SENTINEL.encode()}}, "FlowSpeech headers must be a mapping"),
({"extra_headers": {"Authorization": {"value": SECRET_SENTINEL}}}, "FlowSpeech headers must be a mapping"),
],
)
async def test_invalid_credentials_never_reach_errors_or_logs(asynchronous, invalid, message, monkeypatch, caplog, capsys):
caplog.set_level(logging.DEBUG, logger="LiteLLM")
recorder = FailureRecorder()
monkeypatch.setattr(litellm, "failure_callback", [recorder])
monkeypatch.setattr(litellm, "_async_failure_callback", [recorder])
requests = []
def respond(request):
requests.append(request)
return httpx.Response(200, json=success())
kwargs = {"model": "flowspeech/flowspeech-tts", "input": "Hello", "api_key": "test-key", **invalid}
if asynchronous:
async with httpx.AsyncClient(transport=httpx.MockTransport(respond)) as client:
with pytest.raises(litellm.APIConnectionError, match=message) as error:
await litellm.aspeech(**kwargs, client=client)
else:
with httpx.Client(transport=httpx.MockTransport(respond)) as client:
with pytest.raises(ValueError, match=message) as error:
litellm.speech(**kwargs, client=client)
captured = capsys.readouterr()
assert requests == []
assert recorder.errors
assert SECRET_SENTINEL not in str(error.value)
assert SECRET_SENTINEL not in "".join(traceback.format_exception(error.value))
assert SECRET_SENTINEL not in repr(recorder.errors)
assert SECRET_SENTINEL not in caplog.text + captured.out + captured.err
def test_registers_provider():
from litellm.utils import ProviderConfigManager