fix(realtime): map OpenAI TTS voices before the Live voice check, keep outage spend rows, share the playground session config
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled

The Live voice gate now judges the Gemini voice an OpenAI TTS name maps to, so fable and onyx reach Vertex as Umbriel and Orus again. An upstream close before the held setup was sent is logged as a session, not a refusal. The playground sends one server_vad session config for typed and voice turns, and the e2e, integration, and dashboard tests follow lazy setup.
This commit is contained in:
mateo-berri 2026-10-29 15:20:20 -07:00
parent ec7569040d
commit 6b58477f86
9 changed files with 104 additions and 55 deletions

View file

@ -1568,7 +1568,11 @@ class RealTimeStreaming:
return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
def _backend_refused_session(self, close: BackendClose) -> bool:
return close.code != 1000 and not self._backend_sent_frames
return (
close.code != 1000
and not self._backend_sent_frames
and not self._holds_session_updates_until_first_content()
)
async def _log_session_end(self, transport_refusal: Exception | None) -> None:
provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal()

View file

@ -18,6 +18,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
OPENAI_TO_GEMINI_TTS_VOICES,
VertexGeminiConfig,
)
from litellm.responses.litellm_completion_transformation.transformation import (
@ -125,7 +126,7 @@ def _is_undocumented_live_voice(voice: object) -> bool:
return (
isinstance(voice, str)
and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES
and voice.lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
)

View file

@ -47,6 +47,7 @@ class RealtimeProvider:
id: str
alias: str
litellm_params: LiteLLMParamsBody
modalities: tuple[str, ...] = ("text",)
PROVIDERS = (
@ -75,6 +76,7 @@ PROVIDERS = (
model="gemini/gemini-3.1-flash-live-preview",
api_key="os.environ/GEMINI_API_KEY",
),
modalities=("audio",),
),
RealtimeProvider(
"vertex_ai",
@ -84,6 +86,7 @@ PROVIDERS = (
vertex_location="us-central1",
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
),
modalities=("audio",),
),
RealtimeProvider(
"bedrock",

View file

@ -124,7 +124,8 @@ def test_text_conversation(
session.send(
SessionUpdate(
session=SessionConfig(
instructions="You are a terse assistant. Reply in one short sentence."
modalities=list(provider.modalities),
instructions="You are a terse assistant. Reply in one short sentence.",
)
)
)
@ -159,6 +160,7 @@ def test_tool_call_round_trip(
session.send(
SessionUpdate(
session=SessionConfig(
modalities=list(provider.modalities),
tools=[WEATHER_TOOL],
tool_choice="auto",
instructions=(

View file

@ -355,7 +355,13 @@ def _voice_scenario_created_after_the_first_frame(transcript: str) -> RealtimeRe
def _live_scenario() -> RealtimeResponse:
return RealtimeResponse(content_type="application/x-realtime", events=())
return RealtimeResponse(
content_type="application/x-realtime",
events=(
{"serverContent": {"modelTurn": {"parts": [{"text": "scripted $REQUEST_ID"}]}}},
{"serverContent": {"turnComplete": True}},
),
)
def _realtime_update(turn_detection: JsonValue) -> dict[str, JsonValue]:
@ -1348,10 +1354,14 @@ def test_gemini_ai_studio_voice_session_with_vad_tuning_connects_under_the_guard
_ws_base(_owned_url(guardrail_proxy)),
f"model={model}",
key,
(_step(_update_frame(update), until="session.updated"),),
(
_step(_update_frame(update), until="session.updated"),
_step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE),
),
)
observed: Final = _observed(upstream_url, handle.scenario_id)
assert session.types == ("session.created", "session.updated"), session
assert session.types[:2] == ("session.created", "session.updated"), session
assert session.types[-1] == RESPONSE_DONE, session
assert _setup_detection(observed) == {"disabled": True}, _sent(observed)
@ -1365,10 +1375,14 @@ def test_vertex_live_voice_session_keeps_vad_tuning_under_the_guardrail(guardrai
_ws_base(_owned_url(guardrail_proxy)),
f"model={model}",
key,
(_step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"),),
(
_step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"),
_step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE),
),
)
observed: Final = _observed(upstream_url, handle.scenario_id)
assert session.types == ("session.created", "session.updated"), session
assert session.types[:2] == ("session.created", "session.updated"), session
assert session.types[-1] == RESPONSE_DONE, session
assert _setup_detection(observed) == GEMINI_TUNED_DETECTION, _sent(observed)

View file

@ -4174,6 +4174,24 @@ async def test_upstream_refusal_after_a_synthetic_session_created_still_logs_a_f
assert session.logging.logged_sessions == ()
class _LazySetupRealtimeConfig(_PassthroughRealtimeConfig):
def builds_setup_from_session_update(self) -> bool:
return True
@pytest.mark.asyncio
async def test_upstream_outage_before_a_held_setup_was_sent_logs_the_session_not_a_refusal():
upstream_close: Final = ConnectionClosed(Close(1011, "upstream outage"), None)
session: Final = _relay_session(
_client_ws_that_never_sends(), _backend_ws_closing_with(upstream_close), _LazySetupRealtimeConfig()
)
await session.run()
assert session.logging.logged_failures == ()
assert len(session.logging.logged_sessions) == 1
@pytest.mark.asyncio
async def test_upstream_close_after_relayed_events_still_logs_the_session_as_success():
client_ws: Final = _client_ws_that_never_sends()

View file

@ -18,6 +18,7 @@ import websockets.exceptions # registers websockets.exceptions on the websocket
import litellm
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import OPENAI_TO_GEMINI_TTS_VOICES
from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig
# ---------------------------------------------------------------------------
@ -699,3 +700,20 @@ def test_vertex_drops_an_undocumented_voice_with_drop_params():
json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
)
assert "speechConfig" not in json.loads(messages[0])["setup"].get("generationConfig", {})
@pytest.mark.parametrize("drop_params", [False, True])
@pytest.mark.parametrize("voice", ["fable", "onyx"])
def test_vertex_sends_the_documented_voice_an_openai_tts_voice_maps_to(voice, drop_params):
cfg: Final = VertexAIRealtimeConfig(
access_token="tok", project="my-proj", location="us-central1", drop_params=drop_params
)
session_update: Final = {"type": "session.update", "session": {"voice": voice}}
assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", session_update["session"], setup_sent=False) is None
messages: Final = cfg.transform_realtime_request(
json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
)
assert json.loads(messages[0])["setup"]["generationConfig"]["speechConfig"] == {
"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES[voice]}}
}

View file

@ -173,6 +173,24 @@ describe("RealtimePlayground", () => {
expect(update.session.type).toBe("realtime");
});
it("starts recording on the configuration the session was created with", async () => {
const user = userEvent.setup();
render(<RealtimePlayground {...props} />);
await connect(user);
await act(async () => {
latestSocket().emit({ type: "session.created" });
});
await user.click(screen.getByTitle("Start recording"));
const updates = latestSocket()
.sent.map((raw) => JSON.parse(raw))
.filter((payload) => payload.type === "session.update");
expect(updates).toHaveLength(2);
expect(updates[1]).toEqual(updates[0]);
expect(updates[0].session.turn_detection).toEqual({ type: "server_vad" });
});
it("sends what was typed and then asks for a response", async () => {
const user = userEvent.setup();
render(<RealtimePlayground {...props} />);

View file

@ -22,6 +22,20 @@ interface RealtimePlaygroundProps {
selectedGuardrails?: string[];
}
const sessionUpdate = (voice: string) =>
JSON.stringify({
type: "session.update",
session: {
type: "realtime",
modalities: ["text", "audio"],
voice,
input_audio_format: "pcm16",
output_audio_format: "pcm16",
input_audio_transcription: { model: "gpt-4o-mini-transcribe" },
turn_detection: { type: "server_vad" },
},
});
const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({
accessToken,
selectedModel,
@ -125,21 +139,7 @@ const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({
const type = data.type;
if (type === "session.created") {
// GA: session.type is required ("realtime" | "transcription")
ws.send(
JSON.stringify({
type: "session.update",
session: {
type: "realtime",
modalities: ["text", "audio"],
voice: selectedVoice,
input_audio_format: "pcm16",
output_audio_format: "pcm16",
input_audio_transcription: { model: "gpt-4o-mini-transcribe" },
turn_detection: null,
},
}),
);
ws.send(sessionUpdate(selectedVoice));
} else if (type === "session.updated") {
// session configured
} else if (
@ -231,22 +231,7 @@ const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({
const startRecording = useCallback(async () => {
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return;
// Switch to server VAD mode for voice input
// GA: session.type is required
wsRef.current.send(
JSON.stringify({
type: "session.update",
session: {
type: "realtime",
modalities: ["text", "audio"],
voice: selectedVoice,
input_audio_format: "pcm16",
output_audio_format: "pcm16",
input_audio_transcription: { model: "gpt-4o-mini-transcribe" },
turn_detection: { type: "server_vad" },
},
}),
);
wsRef.current.send(sessionUpdate(selectedVoice));
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
@ -301,7 +286,7 @@ const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({
} catch (err: any) {
addMessage("status", `Microphone error: ${err.message}`);
}
}, [addMessage]);
}, [addMessage, selectedVoice]);
const stopRecording = useCallback(() => {
processorRef.current?.disconnect();
@ -317,21 +302,7 @@ const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return;
if (configureSessionRef.current) return;
configureSessionRef.current = true;
// GA: session.type is required
wsRef.current.send(
JSON.stringify({
type: "session.update",
session: {
type: "realtime",
modalities: ["text", "audio"],
voice: selectedVoice,
input_audio_format: "pcm16",
output_audio_format: "pcm16",
input_audio_transcription: { model: "gpt-4o-mini-transcribe" },
turn_detection: null,
},
}),
);
wsRef.current.send(sessionUpdate(selectedVoice));
}, [selectedVoice]);
const sendTextMessage = useCallback(() => {