mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
fix(bedrock): omit system prompt from audio transcription and report its token usage (#41565)
* fix(bedrock): omit system prompt from audio transcription * fix(bedrock): report Converse token usage on audio transcription Bedrock transcriptions returned usage null and were billed at $0. Map Converse inputTokens, outputTokens and totalTokens into the transcription usage so spend uses the model's per-token price. input_token_details is optional because Converse does not split audio and text tokens. Also switch the updated request-shape test to rstest. * fix(bedrock): default transcription temperature to 0 and price ap-southeast-2 Voxtral regionally Voxtral wrapped some transcripts in quotes at its default sampling temperature. In direct Converse calls in ap-southeast-2, 0 of 48 calls at temperature 0 were framed against 7 of 72 at the default, and instruction wording made no clear difference, so the request now sends temperature 0 when the caller sets none, matching OpenAI's transcription default. Calls in ap-southeast-2 billed at the us-east-1 price. The cost map gains bedrock/ap-southeast-2 Voxtral rows with all four service tiers, the Bedrock transcription response carries the deployment region in its hidden params, and the transcription token cost uses the region-resolved model key. The bridge now builds the response with model_copy instead of assigning usage after construction, and the native wheel test checks the usage the compiled route returns. * fix(bedrock): forward transcription language, reject unsupported formats, and bill env and Tokyo/London regions get_optional_params_transcription had no Bedrock branch, so language, response_format and timestamp_granularities were dropped before the Rust transform ever saw them. Bedrock now forwards language, prompt and temperature, accepts response_format json or text (the only shapes it can return), and answers 400 for srt, vtt, verbose_json or timestamp_granularities unless drop_params is set. The billing region now falls back to AWS_REGION_NAME and then AWS_REGION when aws_region_name is unset, the same order the Rust resolver uses to pick the endpoint, so an env-configured region gets its regional price. Adds ap-northeast-1 and eu-west-2 Voxtral rows (standard, batch, flex and priority) from the AWS Bedrock offer files, the other two regions AWS serves Voxtral audio in. * fix(bedrock): send transcription temperature as a number and name the rejected param The proxy keeps multipart form values as strings, so -F temperature=0.2 reached Converse as "0.2" and Bedrock answered 400. The Bedrock mapper now converts temperature to a float and answers 400 (or drops it under drop_params) when it is not a finite number. get_optional_params_transcription copied prompt and temperature back from the raw call after the Bedrock branch ran, which undid that conversion and any drop. The Bedrock branch now owns every OpenAI transcription param. An unsupported param such as timestamp_granularities now gets a 400 that names it instead of the generic "user/encoding format" message. * refactor(bedrock): return unsupported transcription params as a value and raise at the param step * fix(bedrock): drop the retired max_tokens field from the regional voxtral price rows --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com>
This commit is contained in:
David Koh
GitHub
parent
d9c9d84298
commit
57f05e56f6
11 changed files with 516 additions and 37 deletions
|
|
@ -1,14 +1,31 @@
|
|||
use serde_json::Value;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
pub struct AudioTranscriptionUsage {
|
||||
pub input_tokens: u64,
|
||||
pub output_tokens: u64,
|
||||
pub total_tokens: u64,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::wire_type)]
|
||||
pub struct AudioTranscriptionResponseData {
|
||||
pub text: String,
|
||||
pub usage: Option<AudioTranscriptionUsage>,
|
||||
}
|
||||
|
||||
impl AudioTranscriptionResponseData {
|
||||
pub fn into_json(self) -> Value {
|
||||
serde_json::json!({
|
||||
"text": self.text,
|
||||
})
|
||||
match self.usage {
|
||||
None => serde_json::json!({"text": self.text}),
|
||||
Some(usage) => serde_json::json!({
|
||||
"text": self.text,
|
||||
"usage": {
|
||||
"type": "tokens",
|
||||
"input_tokens": usage.input_tokens,
|
||||
"output_tokens": usage.output_tokens,
|
||||
"total_tokens": usage.total_tokens,
|
||||
},
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,8 +6,8 @@ use litellm_auth_aws::{
|
|||
};
|
||||
use litellm_core_utils::core_helpers::json_type_name;
|
||||
use litellm_llms_types::{
|
||||
formats::audio_transcription::AudioTranscriptionResponseData,
|
||||
providers::bedrock::ConverseResponse,
|
||||
formats::audio_transcription::{AudioTranscriptionResponseData, AudioTranscriptionUsage},
|
||||
providers::bedrock::{ConverseResponse, ConverseUsage},
|
||||
};
|
||||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
|
@ -101,10 +101,10 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
if let Some(prompt) = optional_string(&optional_params, "prompt") {
|
||||
instruction.push_str(&format!(" Additional context: {prompt}"));
|
||||
}
|
||||
let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]);
|
||||
if let Some(temperature) = optional_params.get("temperature") {
|
||||
inference_config.insert("temperature".to_string(), temperature.clone());
|
||||
}
|
||||
let temperature = optional_params
|
||||
.get("temperature")
|
||||
.cloned()
|
||||
.unwrap_or(json!(0));
|
||||
Ok(AudioTranscriptionRequestData {
|
||||
body: json!({
|
||||
"messages": [{
|
||||
|
|
@ -114,8 +114,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
{"text": instruction}
|
||||
]
|
||||
}],
|
||||
"system": [{"text": "You are a transcription assistant."}],
|
||||
"inferenceConfig": inference_config,
|
||||
"inferenceConfig": {"maxTokens": 4096, "temperature": temperature},
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
|
@ -125,6 +124,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
_model: &str,
|
||||
response_json: Value,
|
||||
) -> Result<AudioTranscriptionResponseData, Error> {
|
||||
let reports_usage = response_json.get("usage").is_some();
|
||||
let response: ConverseResponse =
|
||||
serde_json::from_value(response_json).map_err(|error| {
|
||||
Error::InvalidResponse(ErrorDetail::invalid("Converse response", error))
|
||||
|
|
@ -137,6 +137,7 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
}
|
||||
Ok(AudioTranscriptionResponseData {
|
||||
text: response.content_text(),
|
||||
usage: reports_usage.then(|| transcription_usage(&response.usage)),
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -191,22 +192,41 @@ impl BaseAudioTranscriptionConfig for BedrockAudioTranscriptionConfig {
|
|||
}
|
||||
}
|
||||
|
||||
fn transcription_usage(usage: &ConverseUsage) -> AudioTranscriptionUsage {
|
||||
AudioTranscriptionUsage {
|
||||
input_tokens: usage.input_tokens,
|
||||
output_tokens: usage.output_tokens,
|
||||
total_tokens: usage
|
||||
.total_tokens
|
||||
.unwrap_or(usage.input_tokens + usage.output_tokens),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rstest::rstest;
|
||||
|
||||
fn no_env(_: &str) -> Option<String> {
|
||||
None
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_matches_python_shape() {
|
||||
let params = Map::from_iter([
|
||||
("language".to_string(), json!("en")),
|
||||
("prompt".to_string(), json!("Speaker names")),
|
||||
("temperature".to_string(), json!(0)),
|
||||
("timestamp_granularities".to_string(), json!(["word"])),
|
||||
]);
|
||||
#[rstest]
|
||||
#[case::caller_temperature(Some(json!(0.4)), json!(0.4))]
|
||||
#[case::default_temperature(None, json!(0))]
|
||||
fn request_sends_audio_and_instruction_without_system_block(
|
||||
#[case] temperature: Option<Value>,
|
||||
#[case] expected_temperature: Value,
|
||||
) {
|
||||
let params = Map::from_iter(
|
||||
[
|
||||
("language".to_string(), json!("en")),
|
||||
("prompt".to_string(), json!("Speaker names")),
|
||||
("timestamp_granularities".to_string(), json!(["word"])),
|
||||
]
|
||||
.into_iter()
|
||||
.chain(temperature.map(|value| ("temperature".to_string(), value))),
|
||||
);
|
||||
let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(¶ms);
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
.transform_audio_transcription_request(
|
||||
|
|
@ -225,22 +245,39 @@ mod tests {
|
|||
{"text": "Transcribe the audio. Respond with only the transcript. The audio language is en. Additional context: Speaker names"}
|
||||
]
|
||||
}],
|
||||
"system": [{"text": "You are a transcription assistant."}],
|
||||
"inferenceConfig": {"maxTokens": 4096, "temperature": 0}
|
||||
"inferenceConfig": {"maxTokens": 4096, "temperature": expected_temperature}
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_concatenates_content_blocks() {
|
||||
#[rstest]
|
||||
#[case::reported_total(
|
||||
json!({"inputTokens": 390, "outputTokens": 60, "totalTokens": 450}),
|
||||
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}})
|
||||
)]
|
||||
#[case::derived_total(
|
||||
json!({"inputTokens": 1, "outputTokens": 2}),
|
||||
json!({"text": "hello world", "usage": {"type": "tokens", "input_tokens": 1, "output_tokens": 2, "total_tokens": 3}})
|
||||
)]
|
||||
fn response_carries_text_and_converse_usage(#[case] usage: Value, #[case] expected: Value) {
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
.transform_audio_transcription_response(
|
||||
"model",
|
||||
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": {"inputTokens": 1, "outputTokens": 2}}),
|
||||
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}, "usage": usage}),
|
||||
)
|
||||
.expect("response");
|
||||
assert_eq!(result.text, "hello world");
|
||||
assert_eq!(result.into_json(), json!({"text": "hello world"}));
|
||||
assert_eq!(result.into_json(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn response_without_usage_reports_none() {
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
.transform_audio_transcription_response(
|
||||
"model",
|
||||
json!({"output": {"message": {"content": [{"text": "hello"}]}}}),
|
||||
)
|
||||
.expect("response");
|
||||
assert_eq!(result.into_json(), json!({"text": "hello"}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -688,7 +688,7 @@ def cost_per_token(
|
|||
elif call_type == "atranscription" or call_type == "transcription":
|
||||
if _transcription_usage_has_token_details(usage_block):
|
||||
return generic_cost_per_token(
|
||||
model=model_without_prefix,
|
||||
model=model,
|
||||
usage=usage_block,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
service_tier=service_tier,
|
||||
|
|
|
|||
|
|
@ -27,13 +27,18 @@ class TranscriptionUsageObjectTransformation:
|
|||
if isinstance(usage_object, TranscriptionUsageDurationObject):
|
||||
return None
|
||||
elif isinstance(usage_object, TranscriptionUsageTokensObject):
|
||||
input_token_details: Final = usage_object.input_token_details
|
||||
return Usage(
|
||||
prompt_tokens=usage_object.input_tokens,
|
||||
completion_tokens=usage_object.output_tokens,
|
||||
total_tokens=usage_object.total_tokens,
|
||||
prompt_tokens_details=PromptTokensDetailsWrapper(
|
||||
text_tokens=usage_object.input_token_details.text_tokens,
|
||||
audio_tokens=usage_object.input_token_details.audio_tokens,
|
||||
prompt_tokens_details=(
|
||||
None
|
||||
if input_token_details is None
|
||||
else PromptTokensDetailsWrapper(
|
||||
text_tokens=input_token_details.text_tokens,
|
||||
audio_tokens=input_token_details.audio_tokens,
|
||||
)
|
||||
),
|
||||
)
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1,7 +1,11 @@
|
|||
import base64
|
||||
import math
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
|
||||
from litellm.rust_bridge import runtime
|
||||
|
|
@ -14,7 +18,72 @@ from litellm.rust_bridge.transcription.native import (
|
|||
RustAtranscription,
|
||||
RustTranscription,
|
||||
)
|
||||
from litellm.types.utils import FileTypes, TranscriptionResponse
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.utils import FileTypes, TranscriptionResponse, TranscriptionUsageTokensObject
|
||||
|
||||
SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS: Final = ("language", "prompt", "temperature", "response_format")
|
||||
TEXT_RESPONSE_FORMATS: Final = frozenset({"json", "text"})
|
||||
|
||||
|
||||
def _as_float(value: object) -> float | None:
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
||||
return None
|
||||
try:
|
||||
number: Final = float(value)
|
||||
except ValueError:
|
||||
return None
|
||||
return number if math.isfinite(number) else None
|
||||
|
||||
|
||||
def _rejection(key: str, value: object) -> str | None:
|
||||
if key not in SUPPORTED_OPENAI_TRANSCRIPTION_PARAMS:
|
||||
return f"bedrock transcription does not support {key}."
|
||||
if key == "response_format" and value not in TEXT_RESPONSE_FORMATS:
|
||||
return f"bedrock transcription returns only the transcript text, so response_format={value!r} is not supported."
|
||||
if key == "temperature" and _as_float(value) is None:
|
||||
return f"bedrock transcription needs a number for temperature, got {value!r}."
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UnsupportedTranscriptionParams:
|
||||
message: str
|
||||
|
||||
|
||||
def map_openai_transcription_params(
|
||||
non_default_params: Mapping[str, object], drop_params: bool
|
||||
) -> Mapping[str, object] | UnsupportedTranscriptionParams:
|
||||
rejections: Final = {
|
||||
key: reason for key, value in non_default_params.items() if (reason := _rejection(key, value)) is not None
|
||||
}
|
||||
if rejections and not drop_params:
|
||||
return UnsupportedTranscriptionParams(
|
||||
message=f"{' '.join(rejections.values())} To drop it from the call, set `litellm.drop_params = True`."
|
||||
)
|
||||
return {
|
||||
key: _as_float(value) if key == "temperature" else value
|
||||
for key, value in non_default_params.items()
|
||||
if key not in rejections
|
||||
}
|
||||
|
||||
|
||||
class _RustTranscriptionResult(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
text: str
|
||||
usage: TranscriptionUsageTokensObject | None = None
|
||||
|
||||
|
||||
def _transcription_response(
|
||||
result: Mapping[str, object], optional_params: Mapping[str, object]
|
||||
) -> TranscriptionResponse:
|
||||
parsed: Final = _RustTranscriptionResult.model_validate(result)
|
||||
response: Final = TranscriptionResponse(text=parsed.text).model_copy(update={"usage": parsed.usage})
|
||||
region_name: Final = (
|
||||
optional_params.get("aws_region_name") or get_secret_str("AWS_REGION_NAME") or get_secret_str("AWS_REGION")
|
||||
)
|
||||
response.hidden_params = {**response.hidden_params, "region_name": region_name}
|
||||
return response
|
||||
|
||||
|
||||
class BedrockAudioTranscriptionRustDispatch:
|
||||
|
|
@ -64,7 +133,7 @@ class BedrockAudioTranscriptionRustDispatch:
|
|||
"timeout_seconds": timeout_to_seconds(timeout),
|
||||
}
|
||||
call: Final = NativeCall(args=(), kwargs=fields, base={})
|
||||
return TranscriptionResponse(**rust(call))
|
||||
return _transcription_response(rust(call), optional_params)
|
||||
|
||||
return runtime.run(
|
||||
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),
|
||||
|
|
@ -97,7 +166,7 @@ class BedrockAudioTranscriptionRustDispatch:
|
|||
"timeout_seconds": timeout_to_seconds(timeout),
|
||||
}
|
||||
call: Final = NativeCall(args=(), kwargs=fields, base={})
|
||||
return TranscriptionResponse(**await rust(call))
|
||||
return _transcription_response(await rust(call), optional_params)
|
||||
|
||||
return await runtime.arun(
|
||||
RouteContext(Route.TRANSCRIPTION, provider=custom_llm_provider, model=model),
|
||||
|
|
|
|||
|
|
@ -41939,6 +41939,120 @@
|
|||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 4.12e-08,
|
||||
"input_cost_per_token_batches": 2.06e-08,
|
||||
"input_cost_per_token_flex": 2.06e-08,
|
||||
"input_cost_per_token_priority": 7.21e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.12e-08,
|
||||
"output_cost_per_token_batches": 2.06e-08,
|
||||
"output_cost_per_token_flex": 2.06e-08,
|
||||
"output_cost_per_token_priority": 7.21e-08,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 1.03e-07,
|
||||
"input_cost_per_token_batches": 5.15e-08,
|
||||
"input_cost_per_token_flex": 5.15e-08,
|
||||
"input_cost_per_token_priority": 1.8025e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.09e-07,
|
||||
"output_cost_per_token_batches": 1.545e-07,
|
||||
"output_cost_per_token_flex": 1.545e-07,
|
||||
"output_cost_per_token_priority": 5.4075e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 5e-08,
|
||||
"input_cost_per_token_batches": 3e-08,
|
||||
"input_cost_per_token_flex": 3e-08,
|
||||
"input_cost_per_token_priority": 9e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-08,
|
||||
"output_cost_per_token_batches": 3e-08,
|
||||
"output_cost_per_token_flex": 3e-08,
|
||||
"output_cost_per_token_priority": 9e-08,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"input_cost_per_token_batches": 6e-08,
|
||||
"input_cost_per_token_flex": 6e-08,
|
||||
"input_cost_per_token_priority": 2.1e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.6e-07,
|
||||
"output_cost_per_token_batches": 1.8e-07,
|
||||
"output_cost_per_token_flex": 1.8e-07,
|
||||
"output_cost_per_token_priority": 6.3e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 6e-08,
|
||||
"input_cost_per_token_batches": 3e-08,
|
||||
"input_cost_per_token_flex": 3e-08,
|
||||
"input_cost_per_token_priority": 1.1e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-08,
|
||||
"output_cost_per_token_batches": 3e-08,
|
||||
"output_cost_per_token_flex": 3e-08,
|
||||
"output_cost_per_token_priority": 1.1e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/eu-west-2/mistral.voxtral-small-24b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 1.6e-07,
|
||||
"input_cost_per_token_batches": 8e-08,
|
||||
"input_cost_per_token_flex": 8e-08,
|
||||
"input_cost_per_token_priority": 2.8e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.7e-07,
|
||||
"output_cost_per_token_batches": 2.3e-07,
|
||||
"output_cost_per_token_flex": 2.3e-07,
|
||||
"output_cost_per_token_priority": 8.2e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"mistral/codestral-2508": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
|
|
|
|||
|
|
@ -2878,7 +2878,7 @@ class TranscriptionUsageTokensObject(LiteLLMBaseModel):
|
|||
input_tokens: int
|
||||
output_tokens: int
|
||||
total_tokens: int
|
||||
input_token_details: TranscriptionUsageInputTokenDetailsObject
|
||||
input_token_details: TranscriptionUsageInputTokenDetailsObject | None = None
|
||||
|
||||
|
||||
class TranscriptionResponse(OpenAIObject):
|
||||
|
|
|
|||
|
|
@ -4067,7 +4067,9 @@ def get_optional_params_transcription(
|
|||
"timestamp_granularities": None,
|
||||
}
|
||||
|
||||
non_default_params = {k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])}
|
||||
non_default_params: Final = {
|
||||
k: v for k, v in passed_params.items() if (k in default_params and v != default_params[k])
|
||||
}
|
||||
optional_params = {}
|
||||
|
||||
## raise exception if non-default value passed for non-openai/azure embedding calls
|
||||
|
|
@ -4104,6 +4106,19 @@ def get_optional_params_transcription(
|
|||
model=model,
|
||||
drop_params=drop_params if drop_params is not None else False,
|
||||
)
|
||||
elif custom_llm_provider == "bedrock":
|
||||
from litellm.llms.bedrock.audio_transcription import (
|
||||
UnsupportedTranscriptionParams,
|
||||
map_openai_transcription_params,
|
||||
)
|
||||
|
||||
match map_openai_transcription_params(
|
||||
non_default_params, drop_params=drop_params is True or litellm.drop_params is True
|
||||
):
|
||||
case UnsupportedTranscriptionParams(message=message):
|
||||
raise UnsupportedParamsError(message=message, llm_provider="bedrock")
|
||||
case mapped_params:
|
||||
optional_params = dict(mapped_params)
|
||||
elif provider_config is not None: # custom audio transcription config
|
||||
supported_params = provider_config.get_supported_openai_params(model=model)
|
||||
_check_valid_arg(supported_params=supported_params)
|
||||
|
|
@ -4118,7 +4133,7 @@ def get_optional_params_transcription(
|
|||
optional_params=optional_params,
|
||||
passed_params=passed_params,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
openai_params=OPENAI_TRANSCRIPTION_PARAMS,
|
||||
openai_params=list(default_params) if custom_llm_provider == "bedrock" else OPENAI_TRANSCRIPTION_PARAMS,
|
||||
additional_drop_params=kwargs.get("additional_drop_params", None),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -41939,6 +41939,120 @@
|
|||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 4.12e-08,
|
||||
"input_cost_per_token_batches": 2.06e-08,
|
||||
"input_cost_per_token_flex": 2.06e-08,
|
||||
"input_cost_per_token_priority": 7.21e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.12e-08,
|
||||
"output_cost_per_token_batches": 2.06e-08,
|
||||
"output_cost_per_token_flex": 2.06e-08,
|
||||
"output_cost_per_token_priority": 7.21e-08,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-southeast-2/mistral.voxtral-small-24b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 1.03e-07,
|
||||
"input_cost_per_token_batches": 5.15e-08,
|
||||
"input_cost_per_token_flex": 5.15e-08,
|
||||
"input_cost_per_token_priority": 1.8025e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.09e-07,
|
||||
"output_cost_per_token_batches": 1.545e-07,
|
||||
"output_cost_per_token_flex": 1.545e-07,
|
||||
"output_cost_per_token_priority": 5.4075e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-southeast-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 5e-08,
|
||||
"input_cost_per_token_batches": 3e-08,
|
||||
"input_cost_per_token_flex": 3e-08,
|
||||
"input_cost_per_token_priority": 9e-08,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-08,
|
||||
"output_cost_per_token_batches": 3e-08,
|
||||
"output_cost_per_token_flex": 3e-08,
|
||||
"output_cost_per_token_priority": 9e-08,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/ap-northeast-1/mistral.voxtral-small-24b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"input_cost_per_token_batches": 6e-08,
|
||||
"input_cost_per_token_flex": 6e-08,
|
||||
"input_cost_per_token_priority": 2.1e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 3.6e-07,
|
||||
"output_cost_per_token_batches": 1.8e-07,
|
||||
"output_cost_per_token_flex": 1.8e-07,
|
||||
"output_cost_per_token_priority": 6.3e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/ap-northeast-1/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/eu-west-2/mistral.voxtral-mini-3b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 6e-08,
|
||||
"input_cost_per_token_batches": 3e-08,
|
||||
"input_cost_per_token_flex": 3e-08,
|
||||
"input_cost_per_token_priority": 1.1e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-08,
|
||||
"output_cost_per_token_batches": 3e-08,
|
||||
"output_cost_per_token_flex": 3e-08,
|
||||
"output_cost_per_token_priority": 1.1e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"bedrock/eu-west-2/mistral.voxtral-small-24b-2507": {
|
||||
"deprecation_date": "2027-03-30",
|
||||
"input_cost_per_token": 1.6e-07,
|
||||
"input_cost_per_token_batches": 8e-08,
|
||||
"input_cost_per_token_flex": 8e-08,
|
||||
"input_cost_per_token_priority": 2.8e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"max_input_tokens": 32768,
|
||||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 4.7e-07,
|
||||
"output_cost_per_token_batches": 2.3e-07,
|
||||
"output_cost_per_token_flex": 2.3e-07,
|
||||
"output_cost_per_token_priority": 8.2e-07,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/eu-west-2/index.json",
|
||||
"supports_audio_input": true,
|
||||
"supports_system_messages": true,
|
||||
"supports_native_structured_output": true
|
||||
},
|
||||
"mistral/codestral-2508": {
|
||||
"cache_read_input_token_cost": 3e-08,
|
||||
"input_cost_per_token": 3e-07,
|
||||
|
|
|
|||
|
|
@ -87,7 +87,7 @@ def assert_native_request(
|
|||
def native_response(status: int) -> bytes:
|
||||
if status == 429:
|
||||
return b'{"error":"native-rate-limit"}'
|
||||
return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}'
|
||||
return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}},"usage":{"inputTokens":390,"outputTokens":60}}'
|
||||
|
||||
|
||||
def load_native(native_path: Path) -> object:
|
||||
|
|
@ -130,6 +130,9 @@ def assert_success(route: str, response: object) -> None:
|
|||
raise TypeError(f"{route} returned {type(response).__name__}, expected dict")
|
||||
if response["text"] != "native-transcription":
|
||||
raise AssertionError(f"{route} returned {response['text']!r}, expected 'native-transcription'")
|
||||
expected_usage: Final = {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450}
|
||||
if response.get("usage") != expected_usage:
|
||||
raise AssertionError(f"{route} returned usage {response.get('usage')!r}, expected {expected_usage!r}")
|
||||
|
||||
|
||||
def assert_rate_limit(route: str, error: BaseException) -> None:
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm.rust_bridge import configuration
|
|||
from litellm.rust_bridge._native import RustBridgeDeclined, RustUpstreamError
|
||||
from litellm.rust_bridge.public_call import NativeCall
|
||||
from litellm.rust_bridge.transcription.native import NATIVE_ATRANSCRIPTION, NATIVE_TRANSCRIPTION
|
||||
from litellm.types.utils import TranscriptionUsageTokensObject
|
||||
|
||||
MODEL: Final = "bedrock/mistral.voxtral-mini-3b-2507"
|
||||
AUDIO_FILE: Final = ("audio.wav", b"audio", "audio/wav")
|
||||
|
|
@ -59,6 +60,14 @@ class AsyncBridge:
|
|||
return {"text": "async rust"}
|
||||
|
||||
|
||||
class UsageBridge:
|
||||
def __call__(self, call: NativeCall) -> dict[str, object]:
|
||||
return {
|
||||
"text": "rust",
|
||||
"usage": {"type": "tokens", "input_tokens": 390, "output_tokens": 60, "total_tokens": 450},
|
||||
}
|
||||
|
||||
|
||||
def dispatch_sync() -> litellm.TranscriptionResponse:
|
||||
return BedrockAudioTranscriptionRustDispatch().audio_transcriptions(
|
||||
model=MODEL,
|
||||
|
|
@ -196,3 +205,99 @@ async def test_async_missing_native_binding_raises_without_python_fallback() ->
|
|||
optional_params={},
|
||||
timeout=None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("region_kwargs", "region_env", "priced_model"),
|
||||
[
|
||||
({}, {}, MODEL),
|
||||
({"aws_region_name": "ap-southeast-2"}, {}, "bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507"),
|
||||
({}, {"AWS_REGION_NAME": "eu-west-2"}, "bedrock/eu-west-2/mistral.voxtral-mini-3b-2507"),
|
||||
({}, {"AWS_REGION": "ap-northeast-1"}, "bedrock/ap-northeast-1/mistral.voxtral-mini-3b-2507"),
|
||||
(
|
||||
{"aws_region_name": "ap-southeast-2"},
|
||||
{"AWS_REGION_NAME": "eu-west-2"},
|
||||
"bedrock/ap-southeast-2/mistral.voxtral-mini-3b-2507",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_bedrock_transcription_bills_the_converse_token_usage(
|
||||
region_kwargs: dict[str, str],
|
||||
region_env: dict[str, str],
|
||||
priced_model: str,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
for name in ("AWS_REGION_NAME", "AWS_REGION"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
for name, value in region_env.items():
|
||||
monkeypatch.setenv(name, value)
|
||||
NATIVE_TRANSCRIPTION.override(UsageBridge())
|
||||
|
||||
response: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, **region_kwargs)
|
||||
|
||||
model_info: Final = litellm.get_model_info(priced_model)
|
||||
expected_cost: Final = 390 * (model_info.get("input_cost_per_token") or 0.0) + 60 * (
|
||||
model_info.get("output_cost_per_token") or 0.0
|
||||
)
|
||||
assert response.usage == TranscriptionUsageTokensObject(
|
||||
type="tokens", input_tokens=390, output_tokens=60, total_tokens=450
|
||||
)
|
||||
assert expected_cost > 0
|
||||
assert litellm.completion_cost(
|
||||
completion_response=response, model=MODEL, call_type="transcription"
|
||||
) == pytest.approx(expected_cost)
|
||||
|
||||
|
||||
def test_bedrock_transcription_forwards_language_and_text_formats_to_rust() -> None:
|
||||
bridge: Final = SyncBridge()
|
||||
NATIVE_TRANSCRIPTION.override(bridge)
|
||||
|
||||
litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", response_format="text", prompt="Names")
|
||||
|
||||
forwarded: Final = bridge.calls[0].resolved["optional_params"]
|
||||
assert {key: forwarded.get(key) for key in ("language", "response_format", "prompt")} == {
|
||||
"language": "fr",
|
||||
"response_format": "text",
|
||||
"prompt": "Names",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("sent", "expected"), [("0.2", 0.2), ("0", 0.0), (0.5, 0.5), (1, 1.0)])
|
||||
def test_bedrock_transcription_forwards_form_temperature_as_a_number(sent: object, expected: float) -> None:
|
||||
bridge: Final = SyncBridge()
|
||||
NATIVE_TRANSCRIPTION.override(bridge)
|
||||
|
||||
litellm.transcription(model=MODEL, file=AUDIO_FILE, temperature=sent)
|
||||
|
||||
forwarded: Final = bridge.calls[0].resolved["optional_params"]["temperature"]
|
||||
assert (type(forwarded), forwarded) == (float, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"unsupported",
|
||||
[
|
||||
{"response_format": "srt"},
|
||||
{"response_format": "verbose_json"},
|
||||
{"timestamp_granularities": ["word"]},
|
||||
{"temperature": "warm"},
|
||||
{"temperature": "inf"},
|
||||
{"temperature": True},
|
||||
],
|
||||
)
|
||||
def test_bedrock_transcription_rejects_params_it_cannot_honor(unsupported: dict[str, object]) -> None:
|
||||
bridge: Final = SyncBridge()
|
||||
NATIVE_TRANSCRIPTION.override(bridge)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError) as raised:
|
||||
litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", **unsupported)
|
||||
|
||||
assert raised.value.status_code == 400
|
||||
assert next(iter(unsupported)) in str(raised.value)
|
||||
assert bridge.calls == ()
|
||||
|
||||
dropped: Final = litellm.transcription(model=MODEL, file=AUDIO_FILE, language="fr", drop_params=True, **unsupported)
|
||||
|
||||
assert dropped.text == "rust"
|
||||
forwarded: Final = bridge.calls[0].resolved["optional_params"]
|
||||
assert forwarded["language"] == "fr"
|
||||
assert forwarded.keys().isdisjoint(unsupported)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue