diff --git a/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs b/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs index 624eb7be4e4..211d4d0f648 100644 --- a/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/audio_transcription/response.rs @@ -1,14 +1,31 @@ use serde_json::Value; +#[macro_rules_attribute::apply(crate::wire_type)] +pub struct AudioTranscriptionUsage { + pub input_tokens: u64, + pub output_tokens: u64, + pub total_tokens: u64, +} + #[macro_rules_attribute::apply(crate::wire_type)] pub struct AudioTranscriptionResponseData { pub text: String, + pub usage: Option, } impl AudioTranscriptionResponseData { pub fn into_json(self) -> Value { - serde_json::json!({ - "text": self.text, - }) + match self.usage { + None => serde_json::json!({"text": self.text}), + Some(usage) => serde_json::json!({ + "text": self.text, + "usage": { + "type": "tokens", + "input_tokens": usage.input_tokens, + "output_tokens": usage.output_tokens, + "total_tokens": usage.total_tokens, + }, + }), + } } } diff --git a/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs index 61dd264336e..aa294b74fab 100644 --- a/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs +++ b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs @@ -6,8 +6,8 @@ use litellm_auth_aws::{ }; use litellm_core_utils::core_helpers::json_type_name; use litellm_llms_types::{ - formats::audio_transcription::AudioTranscriptionResponseData, - providers::bedrock::ConverseResponse, + formats::audio_transcription::{AudioTranscriptionResponseData, AudioTranscriptionUsage}, + providers::bedrock::{ConverseResponse, ConverseUsage}, }; use serde::Deserialize; use serde_json::{Map, Value, json}; @@ -109,10 +109,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { if let Some(prompt) = optional_string(&optional_params, "prompt") { instruction.push_str(&format!(" Additional context: {prompt}")); } - let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]); - if let Some(temperature) = optional_params.get("temperature") { - inference_config.insert("temperature".to_string(), temperature.clone()); - } + let temperature = optional_params + .get("temperature") + .cloned() + .unwrap_or(json!(0)); Ok(AudioTranscriptionRequestData { body: json!({ "messages": [{ @@ -122,8 +122,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { {"text": instruction} ] }], - "system": [{"text": "You are a transcription assistant."}], - "inferenceConfig": inference_config, + "inferenceConfig": {"maxTokens": 4096, "temperature": temperature}, }), }) } @@ -133,6 +132,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { _model: &str, response_json: Value, ) -> Result { + let reports_usage = response_json.get("usage").is_some(); let response: ConverseResponse = serde_json::from_value(response_json).map_err(|error| { Error::InvalidResponse(format!("invalid Converse response: {error}").into()) @@ -146,6 +146,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { } Ok(AudioTranscriptionResponseData { text: response.content_text(), + usage: reports_usage.then(|| transcription_usage(&response.usage)), }) } @@ -200,22 +201,41 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig { } } +fn transcription_usage(usage: &ConverseUsage) -> AudioTranscriptionUsage { + AudioTranscriptionUsage { + input_tokens: usage.input_tokens, + output_tokens: usage.output_tokens, + total_tokens: usage + .total_tokens + .unwrap_or(usage.input_tokens + usage.output_tokens), + } +} + #[cfg(test)] mod tests { use super::*; + use rstest::rstest; fn no_env(_: &str) -> Option { None } - #[test] - fn request_matches_python_shape() { - let params = Map::from_iter([ - ("language".to_string(), json!("en")), - ("prompt".to_string(), json!("Speaker names")), - ("temperature".to_string(), json!(0)), - ("timestamp_granularities".to_string(), json!(["word"])), - ]); + #[rstest] + #[case::caller_temperature(Some(json!(0.4)), json!(0.4))] + #[case::default_temperature(None, json!(0))] + fn request_sends_audio_and_instruction_without_system_block( + #[case] temperature: Option, + #[case] expected_temperature: Value, + ) { + let params = Map::from_iter( + [ + ("language".to_string(), json!("en")), + ("prompt".to_string(), json!("Speaker names")), + ("timestamp_granularities".to_string(), json!(["word"])), + ] + .into_iter() + .chain(temperature.map(|value| ("temperature".to_string(), value))), + ); let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(¶ms); let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG .transform_audio_transcription_request( @@ -234,22 +254,39 @@ mod tests { {"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"} ] }], - "system": [{"text": "You are a transcription assistant."}], - "inferenceConfig": {"maxTokens": 4096, "temperature": 0} + "inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature} }) ); } - #[test] - fn response_concatenates_content_blocks() { + #[rstest] + #[case::reported_total( + json!({"inputTokens": 390, "outputTokens": 60, "totalTokens": 450}), + json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}}) + )] + #[case::derived_total( + json!({"inputTokens": 1, "outputTokens": 2}), + json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 1, "output_tokens": 2, "total_tokens": 3}}) + )] + fn response_carries_text_and_converse_usage(#[case] usage: Value, #[case] expected: Value) { let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG .transform_audio_transcription_response( "model", - json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": {"inputTokens": 1, "outputTokens": 2}}), + json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": usage}), ) .expect("response"); - assert_eq!(result.text, "hello world"); - assert_eq!(result.into_json(), json!({"text": "hello world"})); + assert_eq!(result.into_json(), expected); + } + + #[rstest] + fn response_without_usage_reports_none() { + let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG + .transform_audio_transcription_response( + "model", + json!({"output": {"message": {"content": [{"text": "hello"}]}}}), + ) + .expect("response"); + assert_eq!(result.into_json(), json!({"text": "hello"})); } #[test] diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 550bbfcde98..2b7ee66ac74 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -687,7 +687,7 @@ def cost_per_token( elif call_type == "atranscription" or call_type == "transcription": if _transcription_usage_has_token_details(usage_block): return generic_cost_per_token( - model=model_without_prefix, + model=model, usage=usage_block, custom_llm_provider=custom_llm_provider, service_tier=service_tier, diff --git a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py index a02c40b7611..78a8cb51bac 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py +++ b/litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py @@ -27,13 +27,18 @@ class TranscriptionUsageObjectTransformation: if isinstance(usage_object, TranscriptionUsageDurationObject): return None elif isinstance(usage_object, TranscriptionUsageTokensObject): + input_token_details: Final = usage_object.input_token_details return Usage( prompt_tokens=usage_object.input_tokens, completion_tokens=usage_object.output_tokens, total_tokens=usage_object.total_tokens, - prompt_tokens_details=PromptTokensDetailsWrapper( - text_tokens=usage_object.input_token_details.text_tokens, - audio_tokens=usage_object.input_token_details.audio_tokens, + prompt_tokens_details=( + None + if input_token_details is None + else PromptTokensDetailsWrapper( + text_tokens=input_token_details.text_tokens, + audio_tokens=input_token_details.audio_tokens, + ) ), ) return None diff --git a/litellm/llms/bedrock/audio_transcription/__init__.py b/litellm/llms/bedrock/audio_transcription/__init__.py index 7f7a138398b..6588d75bab6 100644 --- a/litellm/llms/bedrock/audio_transcription/__init__.py +++ b/litellm/llms/bedrock/audio_transcription/__init__.py @@ -1,8 +1,12 @@ import base64 +import math +from collections.abc import Mapping from typing import Final import httpx +from pydantic import BaseModel, ConfigDict +from litellm.exceptions import UnsupportedParamsError from litellm.litellm_core_utils.audio_utils.utils import process_audio_file from litellm.rust_bridge import runtime from litellm.rust_bridge.catalog import Route, RouteContext @@ -14,7 +18,68 @@ from litellm.rust_bridge.transcription.native import ( RustAtranscription, RustTranscription, ) -from litellm.types.utils import FileTypes, TranscriptionResponse +from litellm.secret_managers.main import get_secret_str +from litellm.types.utils import FileTypes, TranscriptionResponse, TranscriptionUsageTokensObject + +SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: Final = ("language", "prompt", "temperature", "response_format") +TEXT_RESPONSE_FORMATS: Final = frozenset({"json", "text"}) + + +def _as_float(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float, str)): + return None + try: + number: Final = float(value) + except ValueError: + return None + return number if math.isfinite(number) else None + + +def _rejection(key: str, value: object) -> str | None: + if key not in SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: + return f"bedrock transcription does not support {key}." + if key == "response_format" and value not in TEXT_RESPONSE_FORMATS: + return f"bedrock transcription returns only the transcript text, so response_format={value!r} is not supported." + if key == "temperature" and _as_float(value) is None: + return f"bedrock transcription needs a number for temperature, got {value!r}." + return None + + +def map_openai_transcription_params( + non_default_params: Mapping[str, object], drop_params: bool +) -> Mapping[str, object]: + rejections: Final = { + key: reason for key, value in non_default_params.items() if (reason := _rejection(key, value)) is not None + } + if rejections and not drop_params: + raise UnsupportedParamsError( + message=f"{' '.join(rejections.values())} To drop it from the call, set `litellm.drop_params = True`.", + llm_provider="bedrock", + ) + return { + key: _as_float(value) if key == "temperature" else value + for key, value in non_default_params.items() + if key not in rejections + } + + +class _RustTranscriptionResult(BaseModel): + model_config = ConfigDict(frozen=True) + + text: str + usage: TranscriptionUsageTokensObject | None = None + + +def _transcription_response( + result: Mapping[str, object], optional_params: Mapping[str, object] +) -> TranscriptionResponse: + parsed: Final = _RustTranscriptionResult.model_validate(result) + response: Final = TranscriptionResponse(text=parsed.text).model_copy(update={"usage": parsed.usage}) + region_name: Final = ( + optional_params.get("aws_region_name") or get_secret_str("AWS_REGION_NAME") or get_secret_str("AWS_REGION") + ) + response.hidden_params = {**response.hidden_params, "region_name": region_name} + return response class BedrockAudioTranscriptionRustDispatch: @@ -64,7 +129,7 @@ class BedrockAudioTranscriptionRustDispatch: "timeout_seconds": timeout_to_seconds(timeout), } call: Final = NativeCall(args=(), kwargs=fields, base={}) - return TranscriptionResponse(**rust(call)) + return _transcription_response(rust(call), optional_params) return runtime.run( RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), @@ -97,7 +162,7 @@ class BedrockAudioTranscriptionRustDispatch: "timeout_seconds": timeout_to_seconds(timeout), } call: Final = NativeCall(args=(), kwargs=fields, base={}) - return TranscriptionResponse(**await rust(call)) + return _transcription_response(await rust(call), optional_params) return await runtime.arun( RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model), diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cb9bfa6ffe4..e418f6fdb86 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -42124,6 +42124,126 @@ "supports_system_messages": true, "supports_native_structured_output": true }, + "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 4.12e-08, + "input_cost_per_token_batches": 2.06e-08, + "input_cost_per_token_flex": 2.06e-08, + "input_cost_per_token_priority": 7.21e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.12e-08, + "output_cost_per_token_batches": 2.06e-08, + "output_cost_per_token_flex": 2.06e-08, + "output_cost_per_token_priority": 7.21e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.03e-07, + "input_cost_per_token_batches": 5.15e-08, + "input_cost_per_token_flex": 5.15e-08, + "input_cost_per_token_priority": 1.8025e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.09e-07, + "output_cost_per_token_batches": 1.545e-07, + "output_cost_per_token_flex": 1.545e-07, + "output_cost_per_token_priority": 5.4075e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 5e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 9e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 5e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 9e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.2e-07, + "input_cost_per_token_batches": 6e-08, + "input_cost_per_token_flex": 6e-08, + "input_cost_per_token_priority": 2.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 6e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 1.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 6e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 1.1e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.6e-07, + "input_cost_per_token_batches": 8e-08, + "input_cost_per_token_flex": 8e-08, + "input_cost_per_token_priority": 2.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.7e-07, + "output_cost_per_token_batches": 2.3e-07, + "output_cost_per_token_flex": 2.3e-07, + "output_cost_per_token_priority": 8.2e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5939a350970..676abb298b0 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2859,7 +2859,7 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel): input_tokens: int output_tokens: int total_tokens: int - input_token_details: TranscriptionUsageInputTokenDetailsObject + input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None class TranscriptionResponse(OpenAIObject): diff --git a/litellm/utils.py b/litellm/utils.py index 7de7ce6188d..12a9f53e431 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -4054,7 +4054,9 @@ def get_optional_params_transcription( "timestamp_granularities": None, } - non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])} + non_default_params: Final = { + k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k]) + } optional_params = {} ## raise exception if non-default value passed for non-openai/azure embedding calls @@ -4091,6 +4093,14 @@ def get_optional_params_transcription( model=model, drop_params=drop_params if drop_params is not None else False, ) + elif custom_llm_provider == "bedrock": + from litellm.llms.bedrock.audio_transcription import map_openai_transcription_params + + optional_params = dict( + map_openai_transcription_params( + non_default_params, drop_params=drop_params is True or litellm.drop_params is True + ) + ) elif provider_config is not None: # custom audio transcription config supported_params = provider_config.get_supported_openai_params(model=model) _check_valid_arg(supported_params=supported_params) @@ -4105,7 +4115,7 @@ def get_optional_params_transcription( optional_params=optional_params, passed_params=passed_params, custom_llm_provider=custom_llm_provider, - openai_params=OPENAI_TRANSCRIPTION_PARAMS, + openai_params=list(default_params) if custom_llm_provider == "bedrock" else OPENAI_TRANSCRIPTION_PARAMS, additional_drop_params=kwargs.get("additional_drop_params", None), ) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cb9bfa6ffe4..e418f6fdb86 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -42124,6 +42124,126 @@ "supports_system_messages": true, "supports_native_structured_output": true }, + "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 4.12e-08, + "input_cost_per_token_batches": 2.06e-08, + "input_cost_per_token_flex": 2.06e-08, + "input_cost_per_token_priority": 7.21e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.12e-08, + "output_cost_per_token_batches": 2.06e-08, + "output_cost_per_token_flex": 2.06e-08, + "output_cost_per_token_priority": 7.21e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.03e-07, + "input_cost_per_token_batches": 5.15e-08, + "input_cost_per_token_flex": 5.15e-08, + "input_cost_per_token_priority": 1.8025e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.09e-07, + "output_cost_per_token_batches": 1.545e-07, + "output_cost_per_token_flex": 1.545e-07, + "output_cost_per_token_priority": 5.4075e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 5e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 9e-08, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 5e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 9e-08, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.2e-07, + "input_cost_per_token_batches": 6e-08, + "input_cost_per_token_flex": 6e-08, + "input_cost_per_token_priority": 2.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 3.6e-07, + "output_cost_per_token_batches": 1.8e-07, + "output_cost_per_token_flex": 1.8e-07, + "output_cost_per_token_priority": 6.3e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 6e-08, + "input_cost_per_token_batches": 3e-08, + "input_cost_per_token_flex": 3e-08, + "input_cost_per_token_priority": 1.1e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 6e-08, + "output_cost_per_token_batches": 3e-08, + "output_cost_per_token_flex": 3e-08, + "output_cost_per_token_priority": 1.1e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, + "bedrock/eu-west-2/mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", + "input_cost_per_token": 1.6e-07, + "input_cost_per_token_batches": 8e-08, + "input_cost_per_token_flex": 8e-08, + "input_cost_per_token_priority": 2.8e-07, + "litellm_provider": "bedrock", + "max_input_tokens": 32768, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4.7e-07, + "output_cost_per_token_batches": 2.3e-07, + "output_cost_per_token_flex": 2.3e-07, + "output_cost_per_token_priority": 8.2e-07, + "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json", + "supports_audio_input": true, + "supports_system_messages": true, + "supports_native_structured_output": true + }, "mistral/codestral-2508": { "cache_read_input_token_cost": 3e-08, "input_cost_per_token": 3e-07, diff --git a/tests/unit/rust_bridge/native_route_wheel_test.py b/tests/unit/rust_bridge/native_route_wheel_test.py index bc3269f5dd9..ed3fe4f6de5 100644 --- a/tests/unit/rust_bridge/native_route_wheel_test.py +++ b/tests/unit/rust_bridge/native_route_wheel_test.py @@ -87,7 +87,7 @@ def assert_native_request( def native_response(status: int) -> bytes: if status == 429: return b'{"error":"native-rate-limit"}' - return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}' + return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}},"usage":{"inputTokens":390,"outputTokens":60}}' def load_native(native_path: Path) -> object: @@ -130,6 +130,9 @@ def assert_success(route: str, response: object) -> None: raise TypeError(f"{route} returned {type(response).__name__}, expected dict") if response["text"] != "native-transcription": raise AssertionError(f"{route} returned {response['text']!r}, expected 'native-transcription'") + expected_usage: Final = {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450} + if response.get("usage") != expected_usage: + raise AssertionError(f"{route} returned usage {response.get('usage')!r}, expected {expected_usage!r}") def assert_rate_limit(route: str, error: BaseException) -> None: diff --git a/tests/unit/test_audio_transcription_rust_bridge.py b/tests/unit/test_audio_transcription_rust_bridge.py index e3a6781dee7..e5d37f227ef 100644 --- a/tests/unit/test_audio_transcription_rust_bridge.py +++ b/tests/unit/test_audio_transcription_rust_bridge.py @@ -11,6 +11,7 @@ from litellm.llms.bedrock.audio_transcription import BedrockAudioTranscriptionRu from litellm.rust_bridge import bindings, configuration from litellm.rust_bridge.public_call import NativeCall from litellm.rust_bridge.transcription.native import NATIVE_ATRANSCRIPTION, NATIVE_TRANSCRIPTION +from litellm.types.utils import TranscriptionUsageTokensObject MODEL: Final = "bedrock/mistral.voxtral-mini-3b-2507" AUDIO_FILE: Final = ("audio.wav", b"audio", "audio/wav") @@ -69,6 +70,14 @@ class AsyncBridge: return {"text": "async rust"} +class UsageBridge: + def __call__(self, call: NativeCall) -> dict[str, object]: + return { + "text": "rust", + "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}, + } + + def dispatch_sync() -> litellm.TranscriptionResponse: return BedrockAudioTranscriptionRustDispatch().audio_transcriptions( model=MODEL, @@ -206,3 +215,99 @@ async def test_async_missing_native_binding_raises_without_python_fallback() -> optional_params={}, timeout=None, ) + + +@pytest.mark.parametrize( + ("region_kwargs", "region_env", "priced_model"), + [ + ({}, {}, MODEL), + ({"aws_region_name": "ap-southeast-2"}, {}, "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507"), + ({}, {"AWS_REGION_NAME": "eu-west-2"}, "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507"), + ({}, {"AWS_REGION": "ap-northeast-1"}, "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507"), + ( + {"aws_region_name": "ap-southeast-2"}, + {"AWS_REGION_NAME": "eu-west-2"}, + "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507", + ), + ], +) +def test_bedrock_transcription_bills_the_converse_token_usage( + region_kwargs: dict[str, str], + region_env: dict[str, str], + priced_model: str, + monkeypatch: pytest.MonkeyPatch, +) -> None: + for name in ("AWS_REGION_NAME", "AWS_REGION"): + monkeypatch.delenv(name, raising=False) + for name, value in region_env.items(): + monkeypatch.setenv(name, value) + NATIVE_TRANSCRIPTION.override(UsageBridge()) + + response: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, **region_kwargs) + + model_info: Final = litellm.get_model_info(priced_model) + expected_cost: Final = 390 * (model_info.get("input_cost_per_token") or 0.0) + 60 * ( + model_info.get("output_cost_per_token") or 0.0 + ) + assert response.usage == TranscriptionUsageTokensObject( + type="tokens", input_tokens=390, output_tokens=60, total_tokens=450 + ) + assert expected_cost > 0 + assert litellm.completion_cost( + completion_response=response, model=MODEL, call_type="transcription" + ) == pytest.approx(expected_cost) + + +def test_bedrock_transcription_forwards_language_and_text_formats_to_rust() -> None: + bridge: Final = SyncBridge() + NATIVE_TRANSCRIPTION.override(bridge) + + litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", response_format="text", prompt="Names") + + forwarded: Final = bridge.calls[0].resolved["optional_params"] + assert {key: forwarded.get(key) for key in ("language", "response_format", "prompt")} == { + "language": "fr", + "response_format": "text", + "prompt": "Names", + } + + +@pytest.mark.parametrize(("sent", "expected"), [("0.2", 0.2), ("0", 0.0), (0.5, 0.5), (1, 1.0)]) +def test_bedrock_transcription_forwards_form_temperature_as_a_number(sent: object, expected: float) -> None: + bridge: Final = SyncBridge() + NATIVE_TRANSCRIPTION.override(bridge) + + litellm.transcription(model=MODEL, file=AUDIO_FILE, temperature=sent) + + forwarded: Final = bridge.calls[0].resolved["optional_params"]["temperature"] + assert (type(forwarded), forwarded) == (float, expected) + + +@pytest.mark.parametrize( + "unsupported", + [ + {"response_format": "srt"}, + {"response_format": "verbose_json"}, + {"timestamp_granularities": ["word"]}, + {"temperature": "warm"}, + {"temperature": "inf"}, + {"temperature": True}, + ], +) +def test_bedrock_transcription_rejects_params_it_cannot_honor(unsupported: dict[str, object]) -> None: + bridge: Final = SyncBridge() + NATIVE_TRANSCRIPTION.override(bridge) + + with pytest.raises(litellm.BadRequestError) as raised: + litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", **unsupported) + + assert raised.value.status_code == 400 + assert next(iter(unsupported)) in str(raised.value) + assert bridge.calls == () + + dropped: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", drop_params=True, **unsupported) + + assert dropped.text == "rust" + forwarded: Final = bridge.calls[0].resolved["optional_params"] + assert forwarded["language"] == "fr" + assert forwarded.keys().isdisjoint(unsupported)