scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 267

fix(bedrock): default transcription temperature to 0 and price ap-southeast-2 Voxtral regionally

Browse source 
Voxtral wrapped some transcripts in quotes at its default sampling
temperature. In direct Converse calls in ap-southeast-2, 0 of 48 calls at
temperature 0 were framed against 7 of 72 at the default, and instruction
wording made no clear difference, so the request now sends temperature 0
when the caller sets none, matching OpenAI's transcription default.

Calls in ap-southeast-2 billed at the us-east-1 price. The cost map gains
bedrock/ap-southeast-2 Voxtral rows with all four service tiers, the Bedrock
transcription response carries the deployment region in its hidden params,
and the transcription token cost uses the region-resolved model key.

The bridge now builds the response with model_copy instead of assigning
usage after construction, and the native wheel test checks the usage the
compiled route returns.
This commit is contained in:
mateo-berri 2026-10-20 20:38:28 -07:00
parent fba25c7752
commit 6104601f5d
7 changed files with 125 additions and 23 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs
View file

@ -109,10 +109,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
if let Some(prompt) = optional_string(&optional_params, "prompt") {
instruction.push_str(&format!(" Additional context: {prompt}"));
}
let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]);
if let Some(temperature) = optional_params.get("temperature") {
inference_config.insert("temperature".to_string(), temperature.clone());
}
let temperature = optional_params
.get("temperature")
.cloned()
.unwrap_or(json!(0));
Ok(AudioTranscriptionRequestData {
body: json!({
"messages": [{
@ -122,7 +122,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
{"text": instruction}
]
}],
"inferenceConfig": inference_config,
"inferenceConfig": {"maxTokens": 4096, "temperature": temperature},
}),
})
}
@ -221,13 +221,21 @@ mod tests {
}
#[rstest]
fn request_sends_audio_and_instruction_without_system_block() {
let params = Map::from_iter([
("language".to_string(), json!("en")),
("prompt".to_string(), json!("Speaker names")),
("temperature".to_string(), json!(0)),
("timestamp_granularities".to_string(), json!(["word"])),
]);
#[case::caller_temperature(Some(json!(0.4)), json!(0.4))]
#[case::default_temperature(None, json!(0))]
fn request_sends_audio_and_instruction_without_system_block(
#[case] temperature: Option<Value>,
#[case] expected_temperature: Value,
) {
let params = Map::from_iter(
[
("language".to_string(), json!("en")),
("prompt".to_string(), json!("Speaker names")),
("timestamp_granularities".to_string(), json!(["word"])),
]
.into_iter()
.chain(temperature.map(|value| ("temperature".to_string(), value))),
);
let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(&params);
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_request(
@ -246,7 +254,7 @@ mod tests {
{"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"}
]
}],
"inferenceConfig": {"maxTokens": 4096, "temperature": 0}
"inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature}
})
);
}

2
litellm/cost_calculator.py
View file

@ -687,7 +687,7 @@ def cost_per_token(
elif call_type == "atranscription" or call_type == "transcription":
if _transcription_usage_has_token_details(usage_block):
return generic_cost_per_token(
model=model_without_prefix,
model=model,
usage=usage_block,
custom_llm_provider=custom_llm_provider,
service_tier=service_tier,

12
litellm/llms/bedrock/audio_transcription/__init__.py
View file

@ -26,10 +26,12 @@ class _RustTranscriptionResult(BaseModel):
usage: TranscriptionUsageTokensObject | None = None
def _transcription_response(result: Mapping[str, object]) -> TranscriptionResponse:
def _transcription_response(
result: Mapping[str, object], optional_params: Mapping[str, object]
) -> TranscriptionResponse:
parsed: Final = _RustTranscriptionResult.model_validate(result)
response: Final = TranscriptionResponse(text=parsed.text)
response.usage = parsed.usage
response: Final = TranscriptionResponse(text=parsed.text).model_copy(update={"usage": parsed.usage})
response.hidden_params = {**response.hidden_params, "region_name": optional_params.get("aws_region_name")}
return response
@ -80,7 +82,7 @@ class BedrockAudioTranscriptionRustDispatch:
"timeout_seconds": timeout_to_seconds(timeout),
}
call: Final = NativeCall(args=(), kwargs=fields, base={})
return _transcription_response(rust(call))
return _transcription_response(rust(call), optional_params)
return runtime.run(
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),
@ -113,7 +115,7 @@ class BedrockAudioTranscriptionRustDispatch:
"timeout_seconds": timeout_to_seconds(timeout),
}
call: Final = NativeCall(args=(), kwargs=fields, base={})
return _transcription_response(await rust(call))
return _transcription_response(await rust(call), optional_params)
return await runtime.arun(
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),

40
litellm/model_prices_and_context_window_backup.json
View file

@ -42124,6 +42124,46 @@
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 4.12e-08,
"input_cost_per_token_batches": 2.06e-08,
"input_cost_per_token_flex": 2.06e-08,
"input_cost_per_token_priority": 7.21e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4.12e-08,
"output_cost_per_token_batches": 2.06e-08,
"output_cost_per_token_flex": 2.06e-08,
"output_cost_per_token_priority": 7.21e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.03e-07,
"input_cost_per_token_batches": 5.15e-08,
"input_cost_per_token_flex": 5.15e-08,
"input_cost_per_token_priority": 1.8025e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.09e-07,
"output_cost_per_token_batches": 1.545e-07,
"output_cost_per_token_flex": 1.545e-07,
"output_cost_per_token_priority": 5.4075e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"mistral/codestral-2508": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,

40
model_prices_and_context_window.json
View file

@ -42124,6 +42124,46 @@
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 4.12e-08,
"input_cost_per_token_batches": 2.06e-08,
"input_cost_per_token_flex": 2.06e-08,
"input_cost_per_token_priority": 7.21e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4.12e-08,
"output_cost_per_token_batches": 2.06e-08,
"output_cost_per_token_flex": 2.06e-08,
"output_cost_per_token_priority": 7.21e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.03e-07,
"input_cost_per_token_batches": 5.15e-08,
"input_cost_per_token_flex": 5.15e-08,
"input_cost_per_token_priority": 1.8025e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.09e-07,
"output_cost_per_token_batches": 1.545e-07,
"output_cost_per_token_flex": 1.545e-07,
"output_cost_per_token_priority": 5.4075e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"mistral/codestral-2508": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,

5
tests/unit/rust_bridge/native_route_wheel_test.py
View file

@ -87,7 +87,7 @@ def assert_native_request(
def native_response(status: int) -> bytes:
if status == 429:
return b'{"error":"native-rate-limit"}'
return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}'
return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}},"usage":{"inputTokens":390,"outputTokens":60}}'
def load_native(native_path: Path) -> object:
@ -130,6 +130,9 @@ def assert_success(route: str, response: object) -> None:
raise TypeError(f"{route} returned {type(response).__name__}, expected dict")
if response["text"] != "native-transcription":
raise AssertionError(f"{route} returned {response['text']!r}, expected 'native-transcription'")
expected_usage: Final = {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}
if response.get("usage") != expected_usage:
raise AssertionError(f"{route} returned usage {response.get('usage')!r}, expected {expected_usage!r}")
def assert_rate_limit(route: str, error: BaseException) -> None:

15
tests/unit/test_audio_transcription_rust_bridge.py
View file

@ -217,12 +217,21 @@ async def test_async_missing_native_binding_raises_without_python_fallback() ->
)
def test_bedrock_transcription_bills_the_converse_token_usage() -> None:
@pytest.mark.parametrize(
("region_kwargs", "priced_model"),
[
({}, MODEL),
({"aws_region_name": "ap-southeast-2"}, "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507"),
],
)
def test_bedrock_transcription_bills_the_converse_token_usage(
region_kwargs: dict[str, str], priced_model: str
) -> None:
NATIVE_TRANSCRIPTION.override(UsageBridge())
response: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE)
response: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, **region_kwargs)
model_info: Final = litellm.get_model_info(MODEL)
model_info: Final = litellm.get_model_info(priced_model)
expected_cost: Final = 390 * (model_info.get("input_cost_per_token") or 0.0) + 60 * (
model_info.get("output_cost_per_token") or 0.0
)