mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-14 23:21:35 +00:00
fix(bedrock): drop system message from voxtral audio transcription request
Bedrock's Converse API rejects a request that carries both a system
block and an audio content block, so every Bedrock Voxtral
transcription call 400s ("Found system messages ... and audio chunks
... This is not allowed prior to the tokenizer version 13."). Confirmed
against live Bedrock that removing the system block, with everything
else identical, makes the same request succeed.
Voxtral already takes its transcription instruction from the user
turn's text block, so the system block was never needed. Update the
existing shape test and add an explicit regression test asserting the
body carries no system key, plus a Python-side test guarding the
dispatch boundary against the same message being reintroduced from
that layer.
This commit is contained in:
parent
c8635ecc67
commit
f015c67f13
2 changed files with 62 additions and 2 deletions
|
|
@ -70,6 +70,12 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig {
|
|||
if let Some(temperature) = optional_params.get("temperature") {
|
||||
inference_config.insert("temperature".to_string(), temperature.clone());
|
||||
}
|
||||
// Bedrock's Converse API rejects a `system` block on any request that
|
||||
// also carries an audio content block (HTTP 400: "Found system
|
||||
// messages ... and audio chunks ... This is not allowed prior to the
|
||||
// tokenizer version 13."). Voxtral takes the transcription
|
||||
// instruction from the user turn's `text` block above, so no system
|
||||
// prompt is needed here.
|
||||
Ok(AudioTranscriptionRequestData {
|
||||
body: json!({
|
||||
"messages": [{
|
||||
|
|
@ -79,7 +85,6 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig {
|
|||
{"text": instruction}
|
||||
]
|
||||
}],
|
||||
"system": [{"text": "You are a transcription assistant."}],
|
||||
"inferenceConfig": inference_config,
|
||||
}),
|
||||
})
|
||||
|
|
@ -179,12 +184,33 @@ mod tests {
|
|||
{"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"}
|
||||
]
|
||||
}],
|
||||
"system": [{"text": "You are a transcription assistant."}],
|
||||
"inferenceConfig": {"maxTokens": 4096, "temperature": 0}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_omits_system_message_alongside_audio_content() {
|
||||
// Regression test: Bedrock's Converse API 400s on a `system` block
|
||||
// combined with an audio content block ("Found system messages ...
|
||||
// and audio chunks ... This is not allowed prior to the tokenizer
|
||||
// version 13."). Confirmed against live Bedrock that dropping
|
||||
// `system` while keeping everything else identical makes the
|
||||
// request succeed.
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
.transform_transcription_request(
|
||||
"mistral.voxtral-mini-3b-2507",
|
||||
json!({"data": "AQI=", "format": "wav", "filename": "sample.wav"}),
|
||||
Map::new(),
|
||||
)
|
||||
.expect("request");
|
||||
assert!(
|
||||
result.body.get("system").is_none(),
|
||||
"request body must not carry a `system` block alongside an audio content block: {}",
|
||||
result.body
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_concatenates_content_blocks() {
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
|
|
|
|||
|
|
@ -149,3 +149,37 @@ async def test_bedrock_atranscription_uses_rust_only_path() -> None:
|
|||
rust_bridge.configure_rust_transcription(transcription=None, atranscription=None)
|
||||
|
||||
assert response.text == "rust"
|
||||
|
||||
|
||||
def test_bedrock_transcription_dispatch_adds_no_system_message() -> None:
|
||||
"""Guard the Python/Rust dispatch boundary for the Bedrock Voxtral fix.
|
||||
|
||||
Bedrock's Converse API rejects a request that carries both a `system`
|
||||
block and an audio content block. The fix removes that `system` block
|
||||
from the request the Rust transform builds (see
|
||||
litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs),
|
||||
since this dispatch layer mocks the whole bridge callable and never sees
|
||||
that internal JSON. This test instead pins the boundary Python does
|
||||
control: it must not hand the bridge a `system` message (directly or via
|
||||
optional_params) that could reintroduce the same conflict from this side.
|
||||
"""
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def capture(**kwargs: object) -> dict[str, object]:
|
||||
captured.update(kwargs)
|
||||
return {"text": "ok"}
|
||||
|
||||
rust_bridge.configure_rust_transcription(transcription=capture, atranscription=None)
|
||||
try:
|
||||
litellm.transcription(
|
||||
model="bedrock/mistral.voxtral-mini-3b-2507",
|
||||
file=("audio.wav", b"audio", "audio/wav"),
|
||||
language="en",
|
||||
)
|
||||
finally:
|
||||
rust_bridge.configure_rust_transcription(transcription=None, atranscription=None)
|
||||
|
||||
assert "system" not in captured
|
||||
optional_params = captured.get("optional_params")
|
||||
assert isinstance(optional_params, dict)
|
||||
assert "system" not in optional_params
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue