mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
fix(realtime): map OpenAI TTS voices before the Live voice check, keep outage spend rows, share the playground session config
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
Some checks failed
Cost map pins recheck / rerun-cost-map-guard (push) Has been cancelled
LiteLLM Rust DB / db-queries (push) Has been cancelled
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Provider / gofmt, vet, build, test (push) Has been cancelled
Terraform Provider / Provider endpoints vs proxy OpenAPI schema (push) Has been cancelled
The Live voice gate now judges the Gemini voice an OpenAI TTS name maps to, so fable and onyx reach Vertex as Umbriel and Orus again. An upstream close before the held setup was sent is logged as a session, not a refusal. The playground sends one server_vad session config for typed and voice turns, and the e2e, integration, and dashboard tests follow lazy setup.
This commit is contained in:
parent
ec7569040d
commit
6b58477f86
9 changed files with 104 additions and 55 deletions
|
|
@ -1568,7 +1568,11 @@ class RealTimeStreaming:
|
|||
return BackendClose(code=1011, reason="proxy failed while relaying the upstream websocket")
|
||||
|
||||
def _backend_refused_session(self, close: BackendClose) -> bool:
|
||||
return close.code != 1000 and not self._backend_sent_frames
|
||||
return (
|
||||
close.code != 1000
|
||||
and not self._backend_sent_frames
|
||||
and not self._holds_session_updates_until_first_content()
|
||||
)
|
||||
|
||||
async def _log_session_end(self, transport_refusal: Exception | None) -> None:
|
||||
provider_refusal: Final = None if self.provider_config is None else self.provider_config.session_refusal()
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
|
|||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeSessionUpdateRejection
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
OPENAI_TO_GEMINI_TTS_VOICES,
|
||||
VertexGeminiConfig,
|
||||
)
|
||||
from litellm.responses.litellm_completion_transformation.transformation import (
|
||||
|
|
@ -125,7 +126,7 @@ def _is_undocumented_live_voice(voice: object) -> bool:
|
|||
return (
|
||||
isinstance(voice, str)
|
||||
and voice.lower() not in OPENAI_STOCK_REALTIME_VOICES
|
||||
and voice.lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
|
||||
and OPENAI_TO_GEMINI_TTS_VOICES.get(voice.lower(), voice).lower() not in GEMINI_LIVE_DOCUMENTED_VOICES
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -47,6 +47,7 @@ class RealtimeProvider:
|
|||
id: str
|
||||
alias: str
|
||||
litellm_params: LiteLLMParamsBody
|
||||
modalities: tuple[str, ...] = ("text",)
|
||||
|
||||
|
||||
PROVIDERS = (
|
||||
|
|
@ -75,6 +76,7 @@ PROVIDERS = (
|
|||
model="gemini/gemini-3.1-flash-live-preview",
|
||||
api_key="os.environ/GEMINI_API_KEY",
|
||||
),
|
||||
modalities=("audio",),
|
||||
),
|
||||
RealtimeProvider(
|
||||
"vertex_ai",
|
||||
|
|
@ -84,6 +86,7 @@ PROVIDERS = (
|
|||
vertex_location="us-central1",
|
||||
vertex_credentials="os.environ/VERTEXAI_CREDENTIALS",
|
||||
),
|
||||
modalities=("audio",),
|
||||
),
|
||||
RealtimeProvider(
|
||||
"bedrock",
|
||||
|
|
|
|||
|
|
@ -124,7 +124,8 @@ def test_text_conversation(
|
|||
session.send(
|
||||
SessionUpdate(
|
||||
session=SessionConfig(
|
||||
instructions="You are a terse assistant. Reply in one short sentence."
|
||||
modalities=list(provider.modalities),
|
||||
instructions="You are a terse assistant. Reply in one short sentence.",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
|
@ -159,6 +160,7 @@ def test_tool_call_round_trip(
|
|||
session.send(
|
||||
SessionUpdate(
|
||||
session=SessionConfig(
|
||||
modalities=list(provider.modalities),
|
||||
tools=[WEATHER_TOOL],
|
||||
tool_choice="auto",
|
||||
instructions=(
|
||||
|
|
|
|||
|
|
@ -355,7 +355,13 @@ def _voice_scenario_created_after_the_first_frame(transcript: str) -> RealtimeRe
|
|||
|
||||
|
||||
def _live_scenario() -> RealtimeResponse:
|
||||
return RealtimeResponse(content_type="application/x-realtime", events=())
|
||||
return RealtimeResponse(
|
||||
content_type="application/x-realtime",
|
||||
events=(
|
||||
{"serverContent": {"modelTurn": {"parts": [{"text": "scripted $REQUEST_ID"}]}}},
|
||||
{"serverContent": {"turnComplete": True}},
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _realtime_update(turn_detection: JsonValue) -> dict[str, JsonValue]:
|
||||
|
|
@ -1348,10 +1354,14 @@ def test_gemini_ai_studio_voice_session_with_vad_tuning_connects_under_the_guard
|
|||
_ws_base(_owned_url(guardrail_proxy)),
|
||||
f"model={model}",
|
||||
key,
|
||||
(_step(_update_frame(update), until="session.updated"),),
|
||||
(
|
||||
_step(_update_frame(update), until="session.updated"),
|
||||
_step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE),
|
||||
),
|
||||
)
|
||||
observed: Final = _observed(upstream_url, handle.scenario_id)
|
||||
assert session.types == ("session.created", "session.updated"), session
|
||||
assert session.types[:2] == ("session.created", "session.updated"), session
|
||||
assert session.types[-1] == RESPONSE_DONE, session
|
||||
assert _setup_detection(observed) == {"disabled": True}, _sent(observed)
|
||||
|
||||
|
||||
|
|
@ -1365,10 +1375,14 @@ def test_vertex_live_voice_session_keeps_vad_tuning_under_the_guardrail(guardrai
|
|||
_ws_base(_owned_url(guardrail_proxy)),
|
||||
f"model={model}",
|
||||
key,
|
||||
(_step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"),),
|
||||
(
|
||||
_step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"),
|
||||
_step(_user_text_item(CLEAN_TRANSCRIPT), RESPONSE_CREATE, until=RESPONSE_DONE),
|
||||
),
|
||||
)
|
||||
observed: Final = _observed(upstream_url, handle.scenario_id)
|
||||
assert session.types == ("session.created", "session.updated"), session
|
||||
assert session.types[:2] == ("session.created", "session.updated"), session
|
||||
assert session.types[-1] == RESPONSE_DONE, session
|
||||
assert _setup_detection(observed) == GEMINI_TUNED_DETECTION, _sent(observed)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -4174,6 +4174,24 @@ async def test_upstream_refusal_after_a_synthetic_session_created_still_logs_a_f
|
|||
assert session.logging.logged_sessions == ()
|
||||
|
||||
|
||||
class _LazySetupRealtimeConfig(_PassthroughRealtimeConfig):
|
||||
def builds_setup_from_session_update(self) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upstream_outage_before_a_held_setup_was_sent_logs_the_session_not_a_refusal():
|
||||
upstream_close: Final = ConnectionClosed(Close(1011, "upstream outage"), None)
|
||||
session: Final = _relay_session(
|
||||
_client_ws_that_never_sends(), _backend_ws_closing_with(upstream_close), _LazySetupRealtimeConfig()
|
||||
)
|
||||
|
||||
await session.run()
|
||||
|
||||
assert session.logging.logged_failures == ()
|
||||
assert len(session.logging.logged_sessions) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_upstream_close_after_relayed_events_still_logs_the_session_as_success():
|
||||
client_ws: Final = _client_ws_that_never_sends()
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import websockets.exceptions # registers websockets.exceptions on the websocket
|
|||
|
||||
import litellm
|
||||
from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import OPENAI_TO_GEMINI_TTS_VOICES
|
||||
from litellm.llms.vertex_ai.realtime.transformation import VertexAIRealtimeConfig
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -699,3 +700,20 @@ def test_vertex_drops_an_undocumented_voice_with_drop_params():
|
|||
json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
|
||||
)
|
||||
assert "speechConfig" not in json.loads(messages[0])["setup"].get("generationConfig", {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("drop_params", [False, True])
|
||||
@pytest.mark.parametrize("voice", ["fable", "onyx"])
|
||||
def test_vertex_sends_the_documented_voice_an_openai_tts_voice_maps_to(voice, drop_params):
|
||||
cfg: Final = VertexAIRealtimeConfig(
|
||||
access_token="tok", project="my-proj", location="us-central1", drop_params=drop_params
|
||||
)
|
||||
session_update: Final = {"type": "session.update", "session": {"voice": voice}}
|
||||
|
||||
assert cfg.setup_session_update_rejection("gemini-live-2.5-flash", session_update["session"], setup_sent=False) is None
|
||||
messages: Final = cfg.transform_realtime_request(
|
||||
json.dumps(session_update), "gemini-live-2.5-flash", session_configuration_request=None
|
||||
)
|
||||
assert json.loads(messages[0])["setup"]["generationConfig"]["speechConfig"] == {
|
||||
"voiceConfig": {"prebuiltVoiceConfig": {"voiceName": OPENAI_TO_GEMINI_TTS_VOICES[voice]}}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -173,6 +173,24 @@ describe("RealtimePlayground", () => {
|
|||
expect(update.session.type).toBe("realtime");
|
||||
});
|
||||
|
||||
it("starts recording on the configuration the session was created with", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<RealtimePlayground {...props} />);
|
||||
|
||||
await connect(user);
|
||||
await act(async () => {
|
||||
latestSocket().emit({ type: "session.created" });
|
||||
});
|
||||
await user.click(screen.getByTitle("Start recording"));
|
||||
|
||||
const updates = latestSocket()
|
||||
.sent.map((raw) => JSON.parse(raw))
|
||||
.filter((payload) => payload.type === "session.update");
|
||||
expect(updates).toHaveLength(2);
|
||||
expect(updates[1]).toEqual(updates[0]);
|
||||
expect(updates[0].session.turn_detection).toEqual({ type: "server_vad" });
|
||||
});
|
||||
|
||||
it("sends what was typed and then asks for a response", async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<RealtimePlayground {...props} />);
|
||||
|
|
|
|||
|
|
@ -22,6 +22,20 @@ interface RealtimePlaygroundProps {
|
|||
selectedGuardrails?: string[];
|
||||
}
|
||||
|
||||
const sessionUpdate = (voice: string) =>
|
||||
JSON.stringify({
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "realtime",
|
||||
modalities: ["text", "audio"],
|
||||
voice,
|
||||
input_audio_format: "pcm16",
|
||||
output_audio_format: "pcm16",
|
||||
input_audio_transcription: { model: "gpt-4o-mini-transcribe" },
|
||||
turn_detection: { type: "server_vad" },
|
||||
},
|
||||
});
|
||||
|
||||
const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({
|
||||
accessToken,
|
||||
selectedModel,
|
||||
|
|
@ -125,21 +139,7 @@ const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({
|
|||
const type = data.type;
|
||||
|
||||
if (type === "session.created") {
|
||||
// GA: session.type is required ("realtime" | "transcription")
|
||||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "realtime",
|
||||
modalities: ["text", "audio"],
|
||||
voice: selectedVoice,
|
||||
input_audio_format: "pcm16",
|
||||
output_audio_format: "pcm16",
|
||||
input_audio_transcription: { model: "gpt-4o-mini-transcribe" },
|
||||
turn_detection: null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
ws.send(sessionUpdate(selectedVoice));
|
||||
} else if (type === "session.updated") {
|
||||
// session configured
|
||||
} else if (
|
||||
|
|
@ -231,22 +231,7 @@ const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({
|
|||
const startRecording = useCallback(async () => {
|
||||
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return;
|
||||
|
||||
// Switch to server VAD mode for voice input
|
||||
// GA: session.type is required
|
||||
wsRef.current.send(
|
||||
JSON.stringify({
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "realtime",
|
||||
modalities: ["text", "audio"],
|
||||
voice: selectedVoice,
|
||||
input_audio_format: "pcm16",
|
||||
output_audio_format: "pcm16",
|
||||
input_audio_transcription: { model: "gpt-4o-mini-transcribe" },
|
||||
turn_detection: { type: "server_vad" },
|
||||
},
|
||||
}),
|
||||
);
|
||||
wsRef.current.send(sessionUpdate(selectedVoice));
|
||||
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
|
|
@ -301,7 +286,7 @@ const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({
|
|||
} catch (err: any) {
|
||||
addMessage("status", `Microphone error: ${err.message}`);
|
||||
}
|
||||
}, [addMessage]);
|
||||
}, [addMessage, selectedVoice]);
|
||||
|
||||
const stopRecording = useCallback(() => {
|
||||
processorRef.current?.disconnect();
|
||||
|
|
@ -317,21 +302,7 @@ const RealtimePlayground: React.FC<RealtimePlaygroundProps> = ({
|
|||
if (!wsRef.current || wsRef.current.readyState !== WebSocket.OPEN) return;
|
||||
if (configureSessionRef.current) return;
|
||||
configureSessionRef.current = true;
|
||||
// GA: session.type is required
|
||||
wsRef.current.send(
|
||||
JSON.stringify({
|
||||
type: "session.update",
|
||||
session: {
|
||||
type: "realtime",
|
||||
modalities: ["text", "audio"],
|
||||
voice: selectedVoice,
|
||||
input_audio_format: "pcm16",
|
||||
output_audio_format: "pcm16",
|
||||
input_audio_transcription: { model: "gpt-4o-mini-transcribe" },
|
||||
turn_detection: null,
|
||||
},
|
||||
}),
|
||||
);
|
||||
wsRef.current.send(sessionUpdate(selectedVoice));
|
||||
}, [selectedVoice]);
|
||||
|
||||
const sendTextMessage = useCallback(() => {
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue