fix(vertex_ai/realtime): normalize all GA-remapped session fields before mapping

Previously _build_vertex_ai_setup_config only lifted nested turn_detection
back to the top level. GA clients' output_modalities and
audio.input.transcription were silently dropped because map_openai_params
only recognises the flat OpenAI-beta keys. Use the parent's
_normalize_session_payload_for_mapping so modalities, transcription, and
turn_detection are all surfaced before mapping.
This commit is contained in:
mateo-berri 2026-05-22 19:43:11 +00:00
parent 2325041913
commit fb935426eb
No known key found for this signature in database
2 changed files with 49 additions and 15 deletions

View file

@ -150,21 +150,13 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
def _build_vertex_ai_setup_config(self, model: str, session_params: dict) -> dict:
"""Build Vertex AI setup configuration with proper model path and defaults."""
# Normalize ``turn_detection`` to the top level so map_openai_params
# picks it up whether the client used the flat beta shape or the
# GA nested shape (session.audio.input.turn_detection). Without
# this, guardrail-injected ``create_response: False`` would be
# silently dropped for GA clients because map_openai_params only
# looks at top-level keys.
extracted_turn_detection = self._extract_turn_detection(session_params)
if (
extracted_turn_detection is not None
and "turn_detection" not in session_params
):
session_params = {
**session_params,
"turn_detection": extracted_turn_detection,
}
# Normalize GA-remapped fields (``output_modalities``, nested
# ``audio.input.transcription``, ``audio.input.turn_detection``) back to
# their flat beta keys so ``map_openai_params`` picks them up. Without
# this, GA clients' explicit modality / transcription / turn-detection
# settings would be silently dropped because ``map_openai_params`` only
# recognises the flat OpenAI-beta key names.
session_params = self._normalize_session_payload_for_mapping(session_params)
setup_config = self.map_openai_params(
optional_params={}, non_default_params=session_params
)

View file

@ -120,6 +120,48 @@ def test_vertex_session_update_defaults_to_audio_modality():
assert setup_payload["generationConfig"]["responseModalities"] == ["AUDIO"]
def test_vertex_session_update_normalizes_ga_remapped_fields():
"""GA-format clients send ``output_modalities`` and nested
``audio.input.transcription`` / ``audio.input.turn_detection``. These must
be normalised back to the flat beta keys before ``map_openai_params``
runs so client preferences aren't silently dropped.
"""
cfg = VertexAIRealtimeConfig(
access_token="tok", project="my-proj", location="us-central1"
)
session_update = {
"type": "session.update",
"session": {
"instructions": "Be concise.",
"output_modalities": ["text"],
"audio": {
"input": {
"transcription": {},
"turn_detection": {"silence_duration_ms": 1500},
},
},
},
}
messages = cfg.transform_realtime_request(
json.dumps(session_update),
"gemini-live-2.5-flash-native-audio",
session_configuration_request=None,
)
assert len(messages) == 1
setup_payload = json.loads(messages[0])["setup"]
assert setup_payload["generationConfig"]["responseModalities"] == ["TEXT"]
assert setup_payload["inputAudioTranscription"] == {}
assert (
setup_payload["realtimeInputConfig"]["automaticActivityDetection"][
"silenceDurationMs"
]
== 1500
)
# ---------------------------------------------------------------------------
# Round-trip test: text-in / text-out via RealTimeStreaming
# ---------------------------------------------------------------------------