scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-28 06:23:07 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 94

Merge ea0d9accb4 into f2118fc767

Browse source
This commit is contained in:
David Koh 2026-10-22 02:12:40 +00:00 • committed by GitHub
commit 8e4e5ae716
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 519 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs
View file

@ -1,14 +1,31 @@
use serde_json::Value;
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct AudioTranscriptionUsage {
pub input_tokens: u64,
pub output_tokens: u64,
pub total_tokens: u64,
}
#[macro_rules_attribute::apply(crate::wire_type)]
pub struct AudioTranscriptionResponseData {
pub text: String,
pub usage: Option<AudioTranscriptionUsage>,
}
impl AudioTranscriptionResponseData {
pub fn into_json(self) -> Value {
serde_json::json!({
"text": self.text,
})
match self.usage {
None => serde_json::json!({"text": self.text}),
Some(usage) => serde_json::json!({
"text": self.text,
"usage": {
"type": "tokens",
"input_tokens": usage.input_tokens,
"output_tokens": usage.output_tokens,
"total_tokens": usage.total_tokens,
},
}),
}
}
}

83
litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs
View file

@ -6,8 +6,8 @@ use litellm_auth_aws::{
};
use litellm_core_utils::core_helpers::json_type_name;
use litellm_llms_types::{
formats::audio_transcription::AudioTranscriptionResponseData,
providers::bedrock::ConverseResponse,
formats::audio_transcription::{AudioTranscriptionResponseData, AudioTranscriptionUsage},
providers::bedrock::{ConverseResponse, ConverseUsage},
};
use serde::Deserialize;
use serde_json::{Map, Value, json};
@ -109,10 +109,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
if let Some(prompt) = optional_string(&optional_params, "prompt") {
instruction.push_str(&format!(" Additional context: {prompt}"));
}
let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]);
if let Some(temperature) = optional_params.get("temperature") {
inference_config.insert("temperature".to_string(), temperature.clone());
}
let temperature = optional_params
.get("temperature")
.cloned()
.unwrap_or(json!(0));
Ok(AudioTranscriptionRequestData {
body: json!({
"messages": [{
@ -122,8 +122,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
{"text": instruction}
]
}],
"system": [{"text": "You are a transcription assistant."}],
"inferenceConfig": inference_config,
"inferenceConfig": {"maxTokens": 4096, "temperature": temperature},
}),
})
}
@ -133,6 +132,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
_model: &str,
response_json: Value,
) -> Result<AudioTranscriptionResponseData, Error> {
let reports_usage = response_json.get("usage").is_some();
let response: ConverseResponse =
serde_json::from_value(response_json).map_err(|error| {
Error::InvalidResponse(format!("invalid Converse response: {error}").into())
@ -146,6 +146,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
}
Ok(AudioTranscriptionResponseData {
text: response.content_text(),
usage: reports_usage.then(|| transcription_usage(&response.usage)),
})
}
@ -200,22 +201,41 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
}
}
fn transcription_usage(usage: &ConverseUsage) -> AudioTranscriptionUsage {
AudioTranscriptionUsage {
input_tokens: usage.input_tokens,
output_tokens: usage.output_tokens,
total_tokens: usage
.total_tokens
.unwrap_or(usage.input_tokens + usage.output_tokens),
}
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
fn no_env(_: &str) -> Option<String> {
None
}
#[test]
fn request_matches_python_shape() {
let params = Map::from_iter([
("language".to_string(), json!("en")),
("prompt".to_string(), json!("Speaker names")),
("temperature".to_string(), json!(0)),
("timestamp_granularities".to_string(), json!(["word"])),
]);
#[rstest]
#[case::caller_temperature(Some(json!(0.4)), json!(0.4))]
#[case::default_temperature(None, json!(0))]
fn request_sends_audio_and_instruction_without_system_block(
#[case] temperature: Option<Value>,
#[case] expected_temperature: Value,
) {
let params = Map::from_iter(
[
("language".to_string(), json!("en")),
("prompt".to_string(), json!("Speaker names")),
("timestamp_granularities".to_string(), json!(["word"])),
]
.into_iter()
.chain(temperature.map(|value| ("temperature".to_string(), value))),
);
let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(&params);
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_request(
@ -234,22 +254,39 @@ mod tests {
{"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"}
]
}],
"system": [{"text": "You are a transcription assistant."}],
"inferenceConfig": {"maxTokens": 4096, "temperature": 0}
"inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature}
})
);
}
#[test]
fn response_concatenates_content_blocks() {
#[rstest]
#[case::reported_total(
json!({"inputTokens": 390, "outputTokens": 60, "totalTokens": 450}),
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}})
)]
#[case::derived_total(
json!({"inputTokens": 1, "outputTokens": 2}),
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 1, "output_tokens": 2, "total_tokens": 3}})
)]
fn response_carries_text_and_converse_usage(#[case] usage: Value, #[case] expected: Value) {
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_response(
"model",
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": {"inputTokens": 1, "outputTokens": 2}}),
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": usage}),
)
.expect("response");
assert_eq!(result.text, "hello world");
assert_eq!(result.into_json(), json!({"text": "hello world"}));
assert_eq!(result.into_json(), expected);
}
#[rstest]
fn response_without_usage_reports_none() {
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_audio_transcription_response(
"model",
json!({"output": {"message": {"content": [{"text": "hello"}]}}}),
)
.expect("response");
assert_eq!(result.into_json(), json!({"text": "hello"}));
}
#[test]

2
litellm/cost_calculator.py
View file

@ -687,7 +687,7 @@ def cost_per_token(
elif call_type == "atranscription" or call_type == "transcription":
if _transcription_usage_has_token_details(usage_block):
return generic_cost_per_token(
model=model_without_prefix,
model=model,
usage=usage_block,
custom_llm_provider=custom_llm_provider,
service_tier=service_tier,

11
litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py
View file

@ -27,13 +27,18 @@ class TranscriptionUsageObjectTransformation:
if isinstance(usage_object, TranscriptionUsageDurationObject):
return None
elif isinstance(usage_object, TranscriptionUsageTokensObject):
input_token_details: Final = usage_object.input_token_details
return Usage(
prompt_tokens=usage_object.input_tokens,
completion_tokens=usage_object.output_tokens,
total_tokens=usage_object.total_tokens,
prompt_tokens_details=PromptTokensDetailsWrapper(
text_tokens=usage_object.input_token_details.text_tokens,
audio_tokens=usage_object.input_token_details.audio_tokens,
prompt_tokens_details=(
None
if input_token_details is None
else PromptTokensDetailsWrapper(
text_tokens=input_token_details.text_tokens,
audio_tokens=input_token_details.audio_tokens,
)
),
)
return None

71
litellm/llms/bedrock/audio_transcription/__init__.py
View file

@ -1,8 +1,12 @@
import base64
import math
from collections.abc import Mapping
from typing import Final
import httpx
from pydantic import BaseModel, ConfigDict
from litellm.exceptions import UnsupportedParamsError
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
from litellm.rust_bridge import runtime
from litellm.rust_bridge.catalog import Route, RouteContext
@ -14,7 +18,68 @@ from litellm.rust_bridge.transcription.native import (
RustAtranscription,
RustTranscription,
)
from litellm.types.utils import FileTypes, TranscriptionResponse
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import FileTypes, TranscriptionResponse, TranscriptionUsageTokensObject
SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: Final = ("language", "prompt", "temperature", "response_format")
TEXT_RESPONSE_FORMATS: Final = frozenset({"json", "text"})
def _as_float(value: object) -> float | None:
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
return None
try:
number: Final = float(value)
except ValueError:
return None
return number if math.isfinite(number) else None
def _rejection(key: str, value: object) -> str | None:
if key not in SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS:
return f"bedrock transcription does not support {key}."
if key == "response_format" and value not in TEXT_RESPONSE_FORMATS:
return f"bedrock transcription returns only the transcript text, so response_format={value!r} is not supported."
if key == "temperature" and _as_float(value) is None:
return f"bedrock transcription needs a number for temperature, got {value!r}."
return None
def map_openai_transcription_params(
non_default_params: Mapping[str, object], drop_params: bool
) -> Mapping[str, object]:
rejections: Final = {
key: reason for key, value in non_default_params.items() if (reason := _rejection(key, value)) is not None
}
if rejections and not drop_params:
raise UnsupportedParamsError(
message=f"{' '.join(rejections.values())} To drop it from the call, set `litellm.drop_params = True`.",
llm_provider="bedrock",
)
return {
key: _as_float(value) if key == "temperature" else value
for key, value in non_default_params.items()
if key not in rejections
}
class _RustTranscriptionResult(BaseModel):
model_config = ConfigDict(frozen=True)
text: str
usage: TranscriptionUsageTokensObject | None = None
def _transcription_response(
result: Mapping[str, object], optional_params: Mapping[str, object]
) -> TranscriptionResponse:
parsed: Final = _RustTranscriptionResult.model_validate(result)
response: Final = TranscriptionResponse(text=parsed.text).model_copy(update={"usage": parsed.usage})
region_name: Final = (
optional_params.get("aws_region_name") or get_secret_str("AWS_REGION_NAME") or get_secret_str("AWS_REGION")
)
response.hidden_params = {**response.hidden_params, "region_name": region_name}
return response
class BedrockAudioTranscriptionRustDispatch:
@ -64,7 +129,7 @@ class BedrockAudioTranscriptionRustDispatch:
"timeout_seconds": timeout_to_seconds(timeout),
}
call: Final = NativeCall(args=(), kwargs=fields, base={})
return TranscriptionResponse(**rust(call))
return _transcription_response(rust(call), optional_params)
return runtime.run(
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),
@ -97,7 +162,7 @@ class BedrockAudioTranscriptionRustDispatch:
"timeout_seconds": timeout_to_seconds(timeout),
}
call: Final = NativeCall(args=(), kwargs=fields, base={})
return TranscriptionResponse(**await rust(call))
return _transcription_response(await rust(call), optional_params)
return await runtime.arun(
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),

120
litellm/model_prices_and_context_window_backup.json
View file

@ -42124,6 +42124,126 @@
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 4.12e-08,
"input_cost_per_token_batches": 2.06e-08,
"input_cost_per_token_flex": 2.06e-08,
"input_cost_per_token_priority": 7.21e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4.12e-08,
"output_cost_per_token_batches": 2.06e-08,
"output_cost_per_token_flex": 2.06e-08,
"output_cost_per_token_priority": 7.21e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.03e-07,
"input_cost_per_token_batches": 5.15e-08,
"input_cost_per_token_flex": 5.15e-08,
"input_cost_per_token_priority": 1.8025e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.09e-07,
"output_cost_per_token_batches": 1.545e-07,
"output_cost_per_token_flex": 1.545e-07,
"output_cost_per_token_priority": 5.4075e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 5e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
"input_cost_per_token_priority": 9e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 5e-08,
"output_cost_per_token_batches": 3e-08,
"output_cost_per_token_flex": 3e-08,
"output_cost_per_token_priority": 9e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.2e-07,
"input_cost_per_token_batches": 6e-08,
"input_cost_per_token_flex": 6e-08,
"input_cost_per_token_priority": 2.1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"output_cost_per_token_batches": 1.8e-07,
"output_cost_per_token_flex": 1.8e-07,
"output_cost_per_token_priority": 6.3e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 6e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
"input_cost_per_token_priority": 1.1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 6e-08,
"output_cost_per_token_batches": 3e-08,
"output_cost_per_token_flex": 3e-08,
"output_cost_per_token_priority": 1.1e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/eu-west-2/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.6e-07,
"input_cost_per_token_batches": 8e-08,
"input_cost_per_token_flex": 8e-08,
"input_cost_per_token_priority": 2.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4.7e-07,
"output_cost_per_token_batches": 2.3e-07,
"output_cost_per_token_flex": 2.3e-07,
"output_cost_per_token_priority": 8.2e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"mistral/codestral-2508": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,

2
litellm/types/utils.py
View file

@ -2859,7 +2859,7 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel):
input_tokens: int
output_tokens: int
total_tokens: int
input_token_details: TranscriptionUsageInputTokenDetailsObject
input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None
class TranscriptionResponse(OpenAIObject):

14
litellm/utils.py
View file

@ -4054,7 +4054,9 @@ def get_optional_params_transcription(
"timestamp_granularities": None,
}
non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])}
non_default_params: Final = {
k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])
}
optional_params = {}
## raise exception if non-default value passed for non-openai/azure embedding calls
@ -4091,6 +4093,14 @@ def get_optional_params_transcription(
model=model,
drop_params=drop_params if drop_params is not None else False,
)
elif custom_llm_provider == "bedrock":
from litellm.llms.bedrock.audio_transcription import map_openai_transcription_params
optional_params = dict(
map_openai_transcription_params(
non_default_params, drop_params=drop_params is True or litellm.drop_params is True
)
)
elif provider_config is not None: # custom audio transcription config
supported_params = provider_config.get_supported_openai_params(model=model)
_check_valid_arg(supported_params=supported_params)
@ -4105,7 +4115,7 @@ def get_optional_params_transcription(
optional_params=optional_params,
passed_params=passed_params,
custom_llm_provider=custom_llm_provider,
openai_params=OPENAI_TRANSCRIPTION_PARAMS,
openai_params=list(default_params) if custom_llm_provider == "bedrock" else OPENAI_TRANSCRIPTION_PARAMS,
additional_drop_params=kwargs.get("additional_drop_params", None),
)

120
model_prices_and_context_window.json
View file

@ -42124,6 +42124,126 @@
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 4.12e-08,
"input_cost_per_token_batches": 2.06e-08,
"input_cost_per_token_flex": 2.06e-08,
"input_cost_per_token_priority": 7.21e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4.12e-08,
"output_cost_per_token_batches": 2.06e-08,
"output_cost_per_token_flex": 2.06e-08,
"output_cost_per_token_priority": 7.21e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.03e-07,
"input_cost_per_token_batches": 5.15e-08,
"input_cost_per_token_flex": 5.15e-08,
"input_cost_per_token_priority": 1.8025e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.09e-07,
"output_cost_per_token_batches": 1.545e-07,
"output_cost_per_token_flex": 1.545e-07,
"output_cost_per_token_priority": 5.4075e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 5e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
"input_cost_per_token_priority": 9e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 5e-08,
"output_cost_per_token_batches": 3e-08,
"output_cost_per_token_flex": 3e-08,
"output_cost_per_token_priority": 9e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.2e-07,
"input_cost_per_token_batches": 6e-08,
"input_cost_per_token_flex": 6e-08,
"input_cost_per_token_priority": 2.1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 3.6e-07,
"output_cost_per_token_batches": 1.8e-07,
"output_cost_per_token_flex": 1.8e-07,
"output_cost_per_token_priority": 6.3e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 6e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
"input_cost_per_token_priority": 1.1e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 6e-08,
"output_cost_per_token_batches": 3e-08,
"output_cost_per_token_flex": 3e-08,
"output_cost_per_token_priority": 1.1e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"bedrock/eu-west-2/mistral.voxtral-small-24b-2507": {
"deprecation_date": "2027-03-30",
"input_cost_per_token": 1.6e-07,
"input_cost_per_token_batches": 8e-08,
"input_cost_per_token_flex": 8e-08,
"input_cost_per_token_priority": 2.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 32768,
"max_output_tokens": 8192,
"max_tokens": 8192,
"mode": "chat",
"output_cost_per_token": 4.7e-07,
"output_cost_per_token_batches": 2.3e-07,
"output_cost_per_token_flex": 2.3e-07,
"output_cost_per_token_priority": 8.2e-07,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
"supports_audio_input": true,
"supports_system_messages": true,
"supports_native_structured_output": true
},
"mistral/codestral-2508": {
"cache_read_input_token_cost": 3e-08,
"input_cost_per_token": 3e-07,

5
tests/unit/rust_bridge/native_route_wheel_test.py
View file

@ -87,7 +87,7 @@ def assert_native_request(
def native_response(status: int) -> bytes:
if status == 429:
return b'{"error":"native-rate-limit"}'
return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}'
return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}},"usage":{"inputTokens":390,"outputTokens":60}}'
def load_native(native_path: Path) -> object:
@ -130,6 +130,9 @@ def assert_success(route: str, response: object) -> None:
raise TypeError(f"{route} returned {type(response).__name__}, expected dict")
if response["text"] != "native-transcription":
raise AssertionError(f"{route} returned {response['text']!r}, expected 'native-transcription'")
expected_usage: Final = {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}
if response.get("usage") != expected_usage:
raise AssertionError(f"{route} returned usage {response.get('usage')!r}, expected {expected_usage!r}")
def assert_rate_limit(route: str, error: BaseException) -> None:

105
tests/unit/test_audio_transcription_rust_bridge.py
View file

@ -11,6 +11,7 @@ from litellm.llms.bedrock.audio_transcription import BedrockAudioTranscriptionRu
from litellm.rust_bridge import bindings, configuration
from litellm.rust_bridge.public_call import NativeCall
from litellm.rust_bridge.transcription.native import NATIVE_ATRANSCRIPTION, NATIVE_TRANSCRIPTION
from litellm.types.utils import TranscriptionUsageTokensObject
MODEL: Final = "bedrock/mistral.voxtral-mini-3b-2507"
AUDIO_FILE: Final = ("audio.wav", b"audio", "audio/wav")
@ -69,6 +70,14 @@ class AsyncBridge:
return {"text": "async rust"}
class UsageBridge:
def __call__(self, call: NativeCall) -> dict[str, object]:
return {
"text": "rust",
"usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450},
}
def dispatch_sync() -> litellm.TranscriptionResponse:
return BedrockAudioTranscriptionRustDispatch().audio_transcriptions(
model=MODEL,
@ -206,3 +215,99 @@ async def test_async_missing_native_binding_raises_without_python_fallback() ->
optional_params={},
timeout=None,
)
@pytest.mark.parametrize(
("region_kwargs", "region_env", "priced_model"),
[
({}, {}, MODEL),
({"aws_region_name": "ap-southeast-2"}, {}, "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507"),
({}, {"AWS_REGION_NAME": "eu-west-2"}, "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507"),
({}, {"AWS_REGION": "ap-northeast-1"}, "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507"),
(
{"aws_region_name": "ap-southeast-2"},
{"AWS_REGION_NAME": "eu-west-2"},
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507",
),
],
)
def test_bedrock_transcription_bills_the_converse_token_usage(
region_kwargs: dict[str, str],
region_env: dict[str, str],
priced_model: str,
monkeypatch: pytest.MonkeyPatch,
) -> None:
for name in ("AWS_REGION_NAME", "AWS_REGION"):
monkeypatch.delenv(name, raising=False)
for name, value in region_env.items():
monkeypatch.setenv(name, value)
NATIVE_TRANSCRIPTION.override(UsageBridge())
response: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, **region_kwargs)
model_info: Final = litellm.get_model_info(priced_model)
expected_cost: Final = 390 * (model_info.get("input_cost_per_token") or 0.0) + 60 * (
model_info.get("output_cost_per_token") or 0.0
)
assert response.usage == TranscriptionUsageTokensObject(
type="tokens", input_tokens=390, output_tokens=60, total_tokens=450
)
assert expected_cost > 0
assert litellm.completion_cost(
completion_response=response, model=MODEL, call_type="transcription"
) == pytest.approx(expected_cost)
def test_bedrock_transcription_forwards_language_and_text_formats_to_rust() -> None:
bridge: Final = SyncBridge()
NATIVE_TRANSCRIPTION.override(bridge)
litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", response_format="text", prompt="Names")
forwarded: Final = bridge.calls[0].resolved["optional_params"]
assert {key: forwarded.get(key) for key in ("language", "response_format", "prompt")} == {
"language": "fr",
"response_format": "text",
"prompt": "Names",
}
@pytest.mark.parametrize(("sent", "expected"), [("0.2", 0.2), ("0", 0.0), (0.5, 0.5), (1, 1.0)])
def test_bedrock_transcription_forwards_form_temperature_as_a_number(sent: object, expected: float) -> None:
bridge: Final = SyncBridge()
NATIVE_TRANSCRIPTION.override(bridge)
litellm.transcription(model=MODEL, file=AUDIO_FILE, temperature=sent)
forwarded: Final = bridge.calls[0].resolved["optional_params"]["temperature"]
assert (type(forwarded), forwarded) == (float, expected)
@pytest.mark.parametrize(
"unsupported",
[
{"response_format": "srt"},
{"response_format": "verbose_json"},
{"timestamp_granularities": ["word"]},
{"temperature": "warm"},
{"temperature": "inf"},
{"temperature": True},
],
)
def test_bedrock_transcription_rejects_params_it_cannot_honor(unsupported: dict[str, object]) -> None:
bridge: Final = SyncBridge()
NATIVE_TRANSCRIPTION.override(bridge)
with pytest.raises(litellm.BadRequestError) as raised:
litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", **unsupported)
assert raised.value.status_code == 400
assert next(iter(unsupported)) in str(raised.value)
assert bridge.calls == ()
dropped: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", drop_params=True, **unsupported)
assert dropped.text == "rust"
forwarded: Final = bridge.calls[0].resolved["optional_params"]
assert forwarded["language"] == "fr"
assert forwarded.keys().isdisjoint(unsupported)