diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime.rs b/litellm-rust/crates/ai-gateway/src/io/realtime.rs index 13b1c86e4dd..4b8213cc8fb 100644 --- a/litellm-rust/crates/ai-gateway/src/io/realtime.rs +++ b/litellm-rust/crates/ai-gateway/src/io/realtime.rs @@ -153,7 +153,7 @@ where // Relay a buffered backend event (warm handoff's session.created) first, so a // warm session looks identical to a fresh one from the client's view. if let Some(event) = prelude { - for outbound in config.transform_realtime_response(&event, model)?.events { + for outbound in config.transform_response(&event, model)?.events { client_out .send(outbound) .await @@ -176,7 +176,7 @@ where // (carrying usage) are server→client events; observing the client arm // would let an authenticated client POST a fabricated response.done and // inflate its own spend log. Logging observes upstream events only. - for outbound in config.transform_realtime_request(&event, model)?.events { + for outbound in config.transform_request(&event, model)?.events { let payload = serde_json::to_string(&outbound) .map_err(|err| Error::InvalidResponse(err.to_string()))?; upstream_tx @@ -193,7 +193,7 @@ where let event: RealtimeEvent = serde_json::from_str(&text) .map_err(|err| Error::InvalidResponse(err.to_string()))?; observe(&event); - for outbound in config.transform_realtime_response(&event, model)?.events { + for outbound in config.transform_response(&event, model)?.events { client_out .send(outbound) .await @@ -212,8 +212,8 @@ where } /// Splice a client realtime stream to OpenAI: forward client events upstream -/// (via `transform_realtime_request`) and backend events downstream (via -/// `transform_realtime_response`). Returns when either side closes. +/// (via `transform_request`) and backend events downstream (via +/// `transform_response`). Returns when either side closes. /// /// Generic over the client transport (typed events) so this crate stays /// framework-agnostic; the gateway adapts its axum socket to these. This is the diff --git a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs index 9a731241e0a..2c9a25b81cc 100644 --- a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs +++ b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs @@ -130,7 +130,7 @@ async fn dial_upstream( api_key: &str, api_base: Option<&str>, ) -> Result { - let url = OPENAI_RESPONSES_WS_CONFIG.complete_websocket_url(api_base, model); + let url = OPENAI_RESPONSES_WS_CONFIG.complete_url(api_base, model); let mut request = url .as_str() .into_client_request() @@ -208,7 +208,7 @@ where event = client_in.next() => { let Some(event) = event else { break }; for outbound in OPENAI_RESPONSES_WS_CONFIG - .transform_ws_request(&event, model)? + .transform_request(&event, model)? .events { let payload = serde_json::to_string(&outbound) @@ -226,7 +226,7 @@ where .map_err(|error| Error::InvalidResponse(error.to_string()))?; observe(&event); for outbound in OPENAI_RESPONSES_WS_CONFIG - .transform_ws_response(&event, model)? + .transform_response(&event, model)? .events { client_out.send(outbound) @@ -265,7 +265,7 @@ where let (mut upstream_tx, upstream_rx) = upstream.split(); if let Some(first_frame) = first_frame { for outbound in OPENAI_RESPONSES_WS_CONFIG - .transform_ws_request(&first_frame, model)? + .transform_request(&first_frame, model)? .events { let payload = serde_json::to_string(&outbound) diff --git a/litellm-rust/crates/core/src/audio_transcription/handler.rs b/litellm-rust/crates/core/src/audio_transcription/handler.rs index a7d65ceb9ad..cc9c9256630 100644 --- a/litellm-rust/crates/core/src/audio_transcription/handler.rs +++ b/litellm-rust/crates/core/src/audio_transcription/handler.rs @@ -1,3 +1,4 @@ +use crate::auth::RequestAuth; use serde_json::Value; use crate::error::Error; @@ -38,7 +39,7 @@ pub async fn execute_audio_transcription_provider_call( .map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?; Ok(request .config - .transform_transcription_response(&request.model, response_json)? + .transform_response(&request.model, response_json)? .into_json()) } @@ -50,11 +51,10 @@ async fn signed_headers( use std::collections::BTreeMap; use std::time::SystemTime; - use crate::audio_transcription::transformation::AudioTranscriptionAuth; use crate::providers::bedrock::audio_transcription::aws_auth_config; use crate::providers::bedrock::aws_base::{resolve_credentials, sign_bedrock_post}; - let AudioTranscriptionAuth::AwsSigV4 { region, .. } = &request.auth else { + let RequestAuth::AwsSigV4 { region, .. } = &request.auth else { return Ok(request.upstream_headers.clone()); }; let env_lookup = |key: &str| std::env::var(key).ok(); @@ -80,12 +80,12 @@ async fn signed_headers( request: &ProviderAudioTranscriptionRequest, _body: &[u8], ) -> Result, Error> { - use crate::audio_transcription::transformation::AudioTranscriptionAuth; - match request.auth { - AudioTranscriptionAuth::AwsSigV4 { .. } => Err(Error::Unsupported( + RequestAuth::AwsSigV4 { .. } => Err(Error::Unsupported( "AWS SigV4 requires the bedrock-auth feature", )), - AudioTranscriptionAuth::Bearer => Ok(request.upstream_headers.clone()), + RequestAuth::Bearer { .. } | RequestAuth::Header { .. } => { + Ok(request.upstream_headers.clone()) + } } } diff --git a/litellm-rust/crates/core/src/audio_transcription/prepare.rs b/litellm-rust/crates/core/src/audio_transcription/prepare.rs index bbef97341a9..35eb3cb4cc4 100644 --- a/litellm-rust/crates/core/src/audio_transcription/prepare.rs +++ b/litellm-rust/crates/core/src/audio_transcription/prepare.rs @@ -1,10 +1,11 @@ +use crate::auth::RequestAuth; use crate::error::Error; use crate::http_utils::{has_header, string_headers}; #[cfg(feature = "bedrock-auth")] use crate::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG; use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; -use super::transformation::{AudioTranscriptionAuth, AudioTranscriptionProviderConfig}; +use super::transformation::AudioTranscriptionProviderConfig; use super::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] @@ -40,12 +41,20 @@ pub fn prepare_audio_transcription_provider_call( .ok_or_else(|| Error::InvalidProvider(provider_info.custom_llm_provider.to_string()))?; let env_lookup = |key: &str| std::env::var(key).ok(); let mut headers = string_headers("audio transcription", request.extra_headers)?; - let auth = config.auth_strategy(&model, &request.optional_params, &env_lookup)?; - if matches!(auth, AudioTranscriptionAuth::Bearer) - && !has_header(&headers, "authorization") - && let Some(api_key) = request.api_key - { - headers.push(("Authorization".to_string(), format!("Bearer {api_key}"))); + let auth = config.auth( + request.api_key, + &model, + &request.optional_params, + &env_lookup, + )?; + match &auth { + RequestAuth::Bearer { token } if !has_header(&headers, "authorization") => { + headers.push(("Authorization".into(), format!("Bearer {token}"))); + } + RequestAuth::Header { name, value } if !has_header(&headers, name) => { + headers.push(((*name).into(), value.clone())); + } + _ => {} } if !has_header(&headers, "content-type") { headers.push(("Content-Type".to_string(), "application/json".to_string())); @@ -56,15 +65,18 @@ pub fn prepare_audio_transcription_provider_call( &request.optional_params, &env_lookup, )?; - let filtered_params = config.map_transcription_params(&request.optional_params); - let transformed = - config.transform_transcription_request(&model, request.audio, filtered_params)?; + let audio = serde_json::from_value(request.audio) + .map_err(|_| Error::InvalidRequest("invalid audio data or format".into()))?; + let params = map_params(request.optional_params.clone())?; + let transformed = config.transform_request(&model, audio, params)?; + let body = serde_json::to_value(transformed) + .map_err(|_| Error::InvalidRequest("invalid audio request body".into()))?; Ok(ProviderAudioTranscriptionRequest { model, custom_llm_provider: provider_info.custom_llm_provider.to_string(), config, url, - body: transformed.body, + body, upstream_headers: headers, auth, #[cfg(feature = "bedrock-auth")] @@ -72,3 +84,11 @@ pub fn prepare_audio_transcription_provider_call( timeout: request.timeout, }) } + +#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] +fn map_params( + params: serde_json::Map, +) -> Result { + serde_json::from_value(serde_json::Value::Object(params)) + .map_err(|_| Error::InvalidRequest("invalid transcription parameters".into())) +} diff --git a/litellm-rust/crates/core/src/audio_transcription/transformation.rs b/litellm-rust/crates/core/src/audio_transcription/transformation.rs index aa9846427dc..529a63bbb32 100644 --- a/litellm-rust/crates/core/src/audio_transcription/transformation.rs +++ b/litellm-rust/crates/core/src/audio_transcription/transformation.rs @@ -1,44 +1,20 @@ use crate::Error; +use crate::auth::RequestAuth; use serde_json::{Map, Value}; -use super::types::{AudioTranscriptionRequestData, AudioTranscriptionResponseData}; - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum AudioTranscriptionAuth { - Bearer, - AwsSigV4 { - region: String, - service: &'static str, - }, -} +use super::types::{ + AudioTranscriptionRequestData, AudioTranscriptionResponseData, ProviderTranscriptionResponse, + TranscriptionAudio, TranscriptionParams, +}; pub trait AudioTranscriptionProviderConfig: Sync { - fn supported_transcription_params(&self) -> &'static [&'static str]; - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn map_transcription_params(&self, params: &Map) -> Map { - params - .iter() - .filter(|(key, _)| { - self.supported_transcription_params() - .contains(&key.as_str()) - }) - .map(|(key, value)| (key.clone(), value.clone())) - .collect() - } - - fn transform_transcription_request( + fn auth( &self, + api_key: Option<&str>, model: &str, - audio: Value, - optional_params: Map, - ) -> Result; - - fn transform_transcription_response( - &self, - model: &str, - response_json: Value, - ) -> Result; + optional_params: &Map, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result; fn complete_url( &self, @@ -48,10 +24,16 @@ pub trait AudioTranscriptionProviderConfig: Sync { env_lookup: &dyn Fn(&str) -> Option, ) -> Result; - fn auth_strategy( + fn transform_request( &self, model: &str, - optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result; + audio: TranscriptionAudio, + params: TranscriptionParams, + ) -> Result; + + fn transform_response( + &self, + model: &str, + response: ProviderTranscriptionResponse, + ) -> Result; } diff --git a/litellm-rust/crates/core/src/audio_transcription/types.rs b/litellm-rust/crates/core/src/audio_transcription/types.rs index 559d7837027..ca64267d17c 100644 --- a/litellm-rust/crates/core/src/audio_transcription/types.rs +++ b/litellm-rust/crates/core/src/audio_transcription/types.rs @@ -1,9 +1,10 @@ +use crate::auth::RequestAuth; use std::time::Duration; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use super::transformation::{AudioTranscriptionAuth, AudioTranscriptionProviderConfig}; +use super::transformation::AudioTranscriptionProviderConfig; pub struct AudioTranscriptionRequest<'a> { pub model: &'a str, @@ -24,7 +25,7 @@ pub struct ProviderAudioTranscriptionRequest { pub(super) url: String, pub(super) body: Value, pub(super) upstream_headers: Vec<(String, String)>, - pub(super) auth: AudioTranscriptionAuth, + pub(super) auth: RequestAuth, #[cfg(feature = "bedrock-auth")] pub(super) optional_params: Map, pub(super) timeout: Option, @@ -52,9 +53,43 @@ impl ProviderAudioTranscriptionRequest { } } -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct AudioTranscriptionRequestData { - pub body: Value, +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum AudioFormat { + Wav, + Mp3, + Flac, + Ogg, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct TranscriptionAudio { + pub data: String, + pub format: AudioFormat, +} + +#[derive(Clone, Debug, Default, Deserialize)] +pub struct TranscriptionParams { + pub language: Option, + pub prompt: Option, + #[serde( + default, + deserialize_with = "crate::http_utils::deserialize_optional_param" + )] + pub temperature: Option>, + pub response_format: Option, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(untagged)] +pub enum AudioTranscriptionRequestData { + Bedrock(crate::providers::bedrock::converse::ConverseRequest), +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum ProviderTranscriptionResponse { + Bedrock(crate::providers::bedrock::converse::ConverseResponse), } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] diff --git a/litellm-rust/crates/core/src/auth/azure/resolve.rs b/litellm-rust/crates/core/src/auth/azure/resolve.rs index eef1f9d6588..83c22addfcd 100644 --- a/litellm-rust/crates/core/src/auth/azure/resolve.rs +++ b/litellm-rust/crates/core/src/auth/azure/resolve.rs @@ -2,7 +2,7 @@ use crate::AuthError; use crate::auth::error::AuthConfigurationError; use crate::auth::{ CredentialFileRef, CredentialLookup, CredentialRef, ResolvedCredential, SecretValue, - TokenCallerHandle, + TokenProviderHandle, }; use super::native::{NativeAzureRequest, NativeAzureTokenAcquirer}; @@ -20,7 +20,7 @@ const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE"; #[derive(Clone, Debug)] pub(crate) enum AzureCredentialPlan { Supplied(ResolvedCredential), - Caller(TokenCallerHandle), + Caller(TokenProviderHandle), Oidc { reference: CredentialRef, tenant_id: String, diff --git a/litellm-rust/crates/core/src/auth/azure/types.rs b/litellm-rust/crates/core/src/auth/azure/types.rs index 72302da64f1..3b80ac9315d 100644 --- a/litellm-rust/crates/core/src/auth/azure/types.rs +++ b/litellm-rust/crates/core/src/auth/azure/types.rs @@ -3,7 +3,7 @@ use serde_json::{Map, Value}; use strum::EnumString; use crate::AuthError; -use crate::auth::{CredentialResolverHandle, SecretValue, TokenCallerHandle}; +use crate::auth::{CredentialResolverHandle, SecretValue, TokenProviderHandle}; pub const DEFAULT_AZURE_SCOPE: &str = "https://cognitiveservices.azure.com/.default"; @@ -36,7 +36,7 @@ pub enum AzureCredentialType { #[derive(Clone, Debug, Default)] pub struct AzureAuthInputs { pub azure_ad_token: ConfigValue, - pub azure_ad_token_provider: Option, + pub azure_ad_token_provider: Option, pub credential_resolver: Option, pub tenant_id: ConfigValue, pub client_id: ConfigValue, diff --git a/litellm-rust/crates/core/src/auth/credential.rs b/litellm-rust/crates/core/src/auth/credential.rs index 185beb7a041..c54c6f80e01 100644 --- a/litellm-rust/crates/core/src/auth/credential.rs +++ b/litellm-rust/crates/core/src/auth/credential.rs @@ -7,7 +7,7 @@ use veil::Redact; use crate::error::AuthError; -use super::{ResolvedCredential, SecretValue, TokenCallerHandle}; +use super::{ResolvedCredential, SecretValue, TokenProviderHandle}; #[derive(Clone, Debug, PartialEq, Eq)] pub enum CredentialFileRef { @@ -55,7 +55,7 @@ impl CredentialResolverHandle { #[derive(Clone, Debug)] pub enum CredentialPlan { Static(CredentialRef), - Caller(TokenCallerHandle), + Caller(TokenProviderHandle), None, } diff --git a/litellm-rust/crates/core/src/auth/http.rs b/litellm-rust/crates/core/src/auth/http.rs index 96ea7168c2e..83931311550 100644 --- a/litellm-rust/crates/core/src/auth/http.rs +++ b/litellm-rust/crates/core/src/auth/http.rs @@ -45,6 +45,15 @@ pub(crate) fn apply_credential( ) } +/// How the upstream call is authenticated. API-key strategies are resolved in +/// `prepare`; SigV4 needs the serialized body, so the handler signs it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RequestAuth { + Header { name: &'static str, value: String }, + Bearer { token: String }, + AwsSigV4 { region: String }, +} + #[cfg(test)] mod tests { use super::{CredentialPlacement, apply_credential}; diff --git a/litellm-rust/crates/core/src/auth/mod.rs b/litellm-rust/crates/core/src/auth/mod.rs index 0f7bc5d60ef..460ba044325 100644 --- a/litellm-rust/crates/core/src/auth/mod.rs +++ b/litellm-rust/crates/core/src/auth/mod.rs @@ -11,7 +11,7 @@ pub use credential::{ CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialPlan, CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle, }; -pub use http::CredentialPlacement; +pub use http::{CredentialPlacement, RequestAuth}; pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy}; pub use secret::SecretValue; -pub use token::{ResolvedCredential, TokenCaller, TokenCallerHandle, TokenFuture}; +pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle}; diff --git a/litellm-rust/crates/core/src/auth/token.rs b/litellm-rust/crates/core/src/auth/token.rs index 922eb9a2041..cfc6b8f0d6b 100644 --- a/litellm-rust/crates/core/src/auth/token.rs +++ b/litellm-rust/crates/core/src/auth/token.rs @@ -29,15 +29,15 @@ impl ResolvedCredential { pub type TokenFuture<'a> = Pin> + Send + 'a>>; -pub trait TokenCaller: std::fmt::Debug + Send + Sync { +pub trait TokenProvider: std::fmt::Debug + Send + Sync { fn acquire(&self) -> TokenFuture<'_>; } #[derive(Clone, Redact)] -pub struct TokenCallerHandle(#[redact(with = "[REDACTED]")] Arc); +pub struct TokenProviderHandle(#[redact(with = "[REDACTED]")] Arc); -impl TokenCallerHandle { - pub fn new(caller: Arc) -> Self { +impl TokenProviderHandle { + pub fn new(caller: Arc) -> Self { Self(caller) } diff --git a/litellm-rust/crates/core/src/chat_completions/common_utils.rs b/litellm-rust/crates/core/src/chat_completions/common_utils.rs index 69e5f175ad5..fb172c6268e 100644 --- a/litellm-rust/crates/core/src/chat_completions/common_utils.rs +++ b/litellm-rust/crates/core/src/chat_completions/common_utils.rs @@ -7,20 +7,52 @@ use super::transformation::ChatCompletionsProviderConfig; const HEADER_CONTEXT: &str = "chat completions"; -#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -pub(super) fn chat_completions_provider_config( - provider: &str, -) -> Option<&'static dyn ChatCompletionsProviderConfig> { +#[derive(Clone, Copy)] +pub(super) enum ChatProviderConfig { + Anthropic, + #[cfg(feature = "bedrock-auth")] + Bedrock, +} + +pub(super) fn chat_completions_provider_config(provider: &str) -> Option { match provider { - "anthropic" => Some(&ANTHROPIC_CHAT_COMPLETIONS_CONFIG), + "anthropic" => Some(ChatProviderConfig::Anthropic), #[cfg(feature = "bedrock-auth")] - "bedrock" => Some( - &crate::providers::bedrock::chat_completions::transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, - ), + "bedrock" => Some(ChatProviderConfig::Bedrock), _ => None, } } +impl ChatProviderConfig { + pub(super) fn decline_reason( + self, + messages: &[super::types::ChatMessage], + params: &Map, + ) -> Option { + match self { + Self::Anthropic => decline_reason(&ANTHROPIC_CHAT_COMPLETIONS_CONFIG, messages, params), + #[cfg(feature = "bedrock-auth")] + Self::Bedrock => decline_reason( + &crate::providers::bedrock::chat_completions::transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, + messages, + params, + ), + } + } +} + +fn decline_reason( + config: &C, + messages: &[super::types::ChatMessage], + params: &Map, +) -> Option { + config.decline_reason(messages, params).or_else(|| { + serde_json::from_value::(Value::Object(params.clone())) + .err() + .map(|_| super::transformation::Unsupported("invalid provider-mapped parameters")) + }) +} + pub(super) fn string_headers( extra_headers: Option>, ) -> Result, Error> { diff --git a/litellm-rust/crates/core/src/chat_completions/conversation.rs b/litellm-rust/crates/core/src/chat_completions/conversation.rs index f7bdc60af37..5536b976541 100644 --- a/litellm-rust/crates/core/src/chat_completions/conversation.rs +++ b/litellm-rust/crates/core/src/chat_completions/conversation.rs @@ -14,7 +14,8 @@ use crate::constants::EMPTY_TEXT_PLACEHOLDER; use super::types::{ChatMessage, ChatMessageContent}; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "lowercase")] pub enum TurnRole { User, Assistant, diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index cf68db5b7ac..24867ea3041 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -1,4 +1,5 @@ use super::error::{ChatRequestError, ChatResponseError}; +use crate::auth::RequestAuth; use crate::error::TransportError; use serde_json::Value; @@ -7,10 +8,9 @@ use crate::http_utils::{http_request, truncate_error_body}; use super::client::http_client; use super::prepare::prepare_provider_request; -use super::transformation::ChatCompletionsAuth; + use super::types::{ - ChatCompletionsResponse, ProviderChatCompletionsRequest, ProviderChatResponseData, - ResolvedChatCompletionsRequest, + ChatCompletionsResponse, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest, }; #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] @@ -49,10 +49,36 @@ pub(super) async fn execute_chat_completions_provider_call( let body: Value = serde_json::from_str(&text) .map_err(|err| ChatResponseError::InvalidJson(err.to_string()))?; - request - .config - .transform_response(&request.model, ProviderChatResponseData { body }) - .map_err(Error::from) + use super::common_utils::ChatProviderConfig; + match request.config { + ChatProviderConfig::Anthropic => decode_response( + &crate::providers::anthropic::chat_completions::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG, + &request.model, + body, + "messages", + ), + #[cfg(feature = "bedrock-auth")] + ChatProviderConfig::Bedrock => decode_response( + &crate::providers::bedrock::chat_completions::transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, + &request.model, + body, + "converse", + ), + }.map_err(Error::from) +} + +pub(crate) fn decode_response( + config: &C, + model: &str, + body: Value, + api: &'static str, +) -> Result { + if !body.is_object() { + return Err(ChatResponseError::NotObject { api }); + } + let response = serde_json::from_value(body) + .map_err(|_| ChatResponseError::InvalidJson("invalid provider response fields".into()))?; + config.transform_response(model, response) } #[cfg(feature = "bedrock-auth")] @@ -68,7 +94,7 @@ pub(super) async fn signed_headers( is_sigv4_computed_header, resolve_credentials, sign_bedrock_post, }; - let ChatCompletionsAuth::AwsSigV4 { region } = &request.auth else { + let RequestAuth::AwsSigV4 { region } = &request.auth else { return Ok(request.upstream_headers.clone()); }; // Reattaching a header the signer also emits would put both copies on the @@ -118,7 +144,7 @@ pub(super) async fn signed_headers( _body: &[u8], ) -> Result, Error> { match &request.auth { - ChatCompletionsAuth::AwsSigV4 { .. } => Err(Error::Unsupported( + RequestAuth::AwsSigV4 { .. } => Err(Error::Unsupported( "AWS SigV4 requires the bedrock-auth feature", )), _ => Ok(request.upstream_headers.clone()), diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index cda5e2b0287..473faf8647f 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -53,7 +53,7 @@ pub fn chat_completions_decline_reason( return Some("empty message list"); } config - .unsupported_reason(&messages, optional_params) + .decline_reason(&messages, optional_params) .map(|reason| reason.0) } diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/core/src/chat_completions/prepare.rs index a28bc02da7d..677134be839 100644 --- a/litellm-rust/crates/core/src/chat_completions/prepare.rs +++ b/litellm-rust/crates/core/src/chat_completions/prepare.rs @@ -1,4 +1,5 @@ use super::error::ChatRequestError; +use crate::auth::RequestAuth; use serde_json::Value; use crate::error::Error; @@ -6,7 +7,7 @@ use crate::http_utils::has_header; use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; use super::common_utils::{chat_completions_provider_config, string_headers}; -use super::transformation::{ChatCompletionsAuth, ChatCompletionsProviderConfig}; +use super::transformation::ChatCompletionsProviderConfig; use super::types::{ ChatCompletionsRequest, ChatMessage, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest, @@ -15,7 +16,7 @@ use super::types::{ pub(super) fn resolve_provider_config<'a>( model: &'a str, custom_llm_provider: Option<&'a str>, -) -> Result<(String, &'static dyn ChatCompletionsProviderConfig), Error> { +) -> Result<(String, super::common_utils::ChatProviderConfig), Error> { let provider_info = get_custom_llm_provider(model, custom_llm_provider) .or_else(|| { custom_llm_provider.map(|provider| CustomLlmProvider { @@ -46,7 +47,7 @@ pub(super) fn resolve_request( if messages.is_empty() { return Err(ChatRequestError::EmptyMessages.into()); } - if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) { + if let Some(reason) = config.decline_reason(&messages, &request.optional_params) { return Err(Error::Unsupported(reason.0)); } Ok(ResolvedChatCompletionsRequest { @@ -62,11 +63,11 @@ pub(super) fn resolve_request( } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] -fn validate_environment( +fn validate_environment( request: &ResolvedChatCompletionsRequest<'_>, model: &str, - config: &dyn ChatCompletionsProviderConfig, -) -> Result<(Vec<(String, String)>, ChatCompletionsAuth), Error> { + config: &C, +) -> Result<(Vec<(String, String)>, RequestAuth), Error> { let env_lookup = |key: &str| std::env::var(key).ok(); let mut headers = string_headers(request.extra_headers.clone())?; let auth = config.auth( @@ -76,7 +77,7 @@ fn validate_environment( &env_lookup, )?; match &auth { - ChatCompletionsAuth::Header { name, value } => { + RequestAuth::Header { name, value } => { // The deployment's credential replaces whatever the caller forwarded // under the same name, mirroring Python's // `{**headers, **anthropic_headers}`: letting a request header win @@ -91,7 +92,7 @@ fn validate_environment( headers.push(((*name).to_string(), value.clone())); } } - ChatCompletionsAuth::Bearer { token } => { + RequestAuth::Bearer { token } => { // Bedrock's `get_request_headers` assigns `headers["Authorization"]` // unconditionally once a bearer token resolves, so the deployment's // identity outranks whatever the caller forwarded. Keeping the @@ -104,7 +105,7 @@ fn validate_environment( headers.push(("authorization".to_string(), format!("Bearer {token}"))); } // SigV4 signs the serialized body, so the handler adds its headers. - ChatCompletionsAuth::AwsSigV4 { .. } => {} + RequestAuth::AwsSigV4 { .. } => {} } for (name, value) in config.default_headers() { @@ -118,24 +119,46 @@ fn validate_environment( pub(super) fn prepare_provider_request( request: ResolvedChatCompletionsRequest<'_>, ) -> Result { - let (headers, auth) = validate_environment(&request, &request.model, request.config)?; - let model = request.model; - let config = request.config; - let env_lookup = |key: &str| std::env::var(key).ok(); + use super::common_utils::ChatProviderConfig; + match request.config { + ChatProviderConfig::Anthropic => prepare_for( + request, + &crate::providers::anthropic::chat_completions::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG, + ), + #[cfg(feature = "bedrock-auth")] + ChatProviderConfig::Bedrock => prepare_for( + request, + &crate::providers::bedrock::chat_completions::transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, + ), + } +} + +fn prepare_for( + request: ResolvedChatCompletionsRequest<'_>, + config: &C, +) -> Result { + let params = + serde_json::from_value::(Value::Object(request.optional_params.clone())) + .map_err(|_| Error::Unsupported("invalid provider-mapped parameters"))?; + let (headers, auth) = validate_environment(&request, &request.model, config)?; let url = config.complete_url( request.api_base, - &model, + &request.model, &request.optional_params, - &env_lookup, + &|key| std::env::var(key).ok(), )?; - let transformed = - config.transform_request(&model, request.messages, request.optional_params.clone())?; - + let transformed = config.transform_request( + &request.model, + super::conversation::build_conversation(&request.messages), + params, + )?; + let body = serde_json::to_value(transformed) + .map_err(|error| ChatRequestError::Serialization(error.to_string()))?; Ok(ProviderChatCompletionsRequest { - model, - config, + model: request.model, + config: request.config, url, - body: transformed.body, + body, upstream_headers: headers, auth, optional_params: request.optional_params, diff --git a/litellm-rust/crates/core/src/chat_completions/transformation.rs b/litellm-rust/crates/core/src/chat_completions/transformation.rs index 633d548566f..8ad6c8b0d96 100644 --- a/litellm-rust/crates/core/src/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/chat_completions/transformation.rs @@ -1,20 +1,11 @@ +use super::conversation::Conversation; use crate::Error; +use crate::auth::RequestAuth; use crate::chat_completions::error::{ChatRequestError, ChatResponseError}; +use serde::{Serialize, de::DeserializeOwned}; use serde_json::{Map, Value}; -use super::types::{ - ChatCompletionsResponse, ChatMessage, ChatMessageContent, ProviderChatRequestData, - ProviderChatResponseData, -}; - -/// How the upstream call is authenticated. API-key strategies are resolved in -/// `prepare`; SigV4 needs the serialized body, so the handler signs it. -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum ChatCompletionsAuth { - Header { name: &'static str, value: String }, - Bearer { token: String }, - AwsSigV4 { region: String }, -} +use super::types::{ChatCompletionsResponse, ChatMessage, ChatMessageContent}; /// Why a request cannot be served by the Rust path. /// @@ -33,13 +24,31 @@ pub const STREAM_PARAM: &str = "stream"; const IGNORABLE_MESSAGE_FIELDS: &[&str] = &["name"]; pub trait ChatCompletionsProviderConfig: Sync { - fn complete_url( + type MappedParams: DeserializeOwned; + type RequestBody: Serialize; + type ResponseBody: DeserializeOwned; + + /// Accepted parameter names after Python provider mapping. + fn supported_provider_params(&self) -> &'static [&'static str]; + + /// Parameters consumed as call configuration (credentials, endpoints) + /// rather than placed in the body. Accepted, never serialized. + fn config_params(&self) -> &'static [&'static str] { + &[] + } + + fn decline_reason( &self, - api_base: Option<&str>, - model: &str, + messages: &[ChatMessage], optional_params: &Map, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result; + ) -> Option { + unsupported_param( + self.supported_provider_params(), + self.config_params(), + optional_params, + ) + .or_else(|| messages.iter().find_map(unsupported_message)) + } fn auth( &self, @@ -47,7 +56,7 @@ pub trait ChatCompletionsProviderConfig: Sync { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> Result; + ) -> Result; fn default_headers(&self) -> &'static [(&'static str, &'static str)] { &[("content-type", "application/json")] @@ -63,44 +72,30 @@ pub trait ChatCompletionsProviderConfig: Sync { false } - /// Supported OpenAI parameter names paired with their provider names. - fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)]; - - /// Parameters consumed as call configuration (credentials, endpoints) - /// rather than placed in the body. Accepted, never serialized. - fn config_params(&self) -> &'static [&'static str] { - &[] - } - - fn unsupported_reason( + fn complete_url( &self, - messages: &[ChatMessage], + api_base: Option<&str>, + model: &str, optional_params: &Map, - ) -> Option { - unsupported_param( - self.supported_openai_params(), - self.config_params(), - optional_params, - ) - .or_else(|| messages.iter().find_map(unsupported_message)) - } + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result; fn transform_request( &self, model: &str, - messages: Vec, - optional_params: Map, - ) -> Result; + conversation: Conversation, + params: Self::MappedParams, + ) -> Result; fn transform_response( &self, model: &str, - response: ProviderChatResponseData, + response: Self::ResponseBody, ) -> Result; } pub fn unsupported_param( - supported: &'static [(&'static str, &'static str)], + supported: &'static [&'static str], config: &'static [&'static str], optional_params: &Map, ) -> Option { @@ -115,9 +110,7 @@ pub fn unsupported_param( .keys() .any(|key| { key != STREAM_PARAM - && !supported - .iter() - .any(|(_, provider_name)| *provider_name == key) + && !supported.iter().any(|provider_name| *provider_name == key) && !config.contains(&key.as_str()) }) .then_some(Unsupported("unrecognized request parameter")) diff --git a/litellm-rust/crates/core/src/chat_completions/types.rs b/litellm-rust/crates/core/src/chat_completions/types.rs index 3238d09b6b5..b9a36ca90a4 100644 --- a/litellm-rust/crates/core/src/chat_completions/types.rs +++ b/litellm-rust/crates/core/src/chat_completions/types.rs @@ -1,10 +1,9 @@ +use crate::auth::RequestAuth; use std::time::Duration; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use super::transformation::{ChatCompletionsAuth, ChatCompletionsProviderConfig}; - /// A `/chat/completions` call as it crosses into the core. /// /// `optional_params` arrives already mapped to the provider's own parameter @@ -24,7 +23,7 @@ pub struct ChatCompletionsRequest<'a> { pub(super) struct ResolvedChatCompletionsRequest<'a> { pub(super) model: String, - pub(super) config: &'static dyn ChatCompletionsProviderConfig, + pub(super) config: super::common_utils::ChatProviderConfig, pub(super) messages: Vec, pub(super) optional_params: Map, pub(super) api_key: Option<&'a str>, @@ -35,28 +34,16 @@ pub(super) struct ResolvedChatCompletionsRequest<'a> { pub(super) struct ProviderChatCompletionsRequest { pub(super) model: String, - pub(super) config: &'static dyn ChatCompletionsProviderConfig, + pub(super) config: super::common_utils::ChatProviderConfig, pub(super) url: String, pub(super) body: Value, pub(super) upstream_headers: Vec<(String, String)>, - pub(super) auth: ChatCompletionsAuth, + pub(super) auth: RequestAuth, #[cfg_attr(not(feature = "bedrock-auth"), allow(dead_code))] pub(super) optional_params: Map, pub(super) timeout: Option, } -/// The provider-shaped request body a config produces. Named rather than a bare -/// `Value` so the transform contract stays a typed one, mirroring -/// [`crate::audio_transcription::types::AudioTranscriptionRequestData`]. -pub struct ProviderChatRequestData { - pub body: Value, -} - -/// The raw provider response body handed back to a config for normalization. -pub struct ProviderChatResponseData { - pub body: Value, -} - #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(untagged)] pub enum ChatMessageContent { diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/core/src/http_utils.rs index cb472dd5a57..6dfbfaf48c0 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/core/src/http_utils.rs @@ -61,6 +61,25 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool { }) } +pub(crate) fn deserialize_optional_param<'de, D, T>( + deserializer: D, +) -> Result>, D::Error> +where + D: serde::Deserializer<'de>, + T: serde::Deserialize<'de>, +{ + as serde::Deserialize>::deserialize(deserializer).map(Some) +} + +pub(crate) fn encode_query_value(value: &str) -> String { + const QUERY_VALUE: &percent_encoding::AsciiSet = &percent_encoding::NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~'); + percent_encoding::utf8_percent_encode(value, QUERY_VALUE).to_string() +} + #[cfg(test)] mod tests { use super::*; diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index ec83d03f535..cb07b2f1280 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,8 +1,9 @@ +use crate::auth::CredentialPlacement; use crate::error::Error; use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider}; use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers}; -use super::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; +use super::transformation::AnthropicMessagesProviderConfig; use super::types::{MessagesRequest, ProviderMessagesRequest}; use serde_json::{Map, Value}; @@ -65,16 +66,16 @@ fn validate_environment( ) -> Result, Error> { let mut headers = string_headers(extra_headers)?; - let auth_strategy = config.auth_strategy(); - let already_authorized = has_header(&headers, auth_strategy.header_name()) + let credential_placement = config.credential_placement(); + let already_authorized = has_header(&headers, credential_placement.header_name()) || (config.accepts_bearer_auth() && has_bearer_auth(&headers)); if !already_authorized { let api_key = config.resolve_api_key(api_key, env_lookup)?; - let auth_header = match auth_strategy { - MessagesAuthStrategy::Bearer => { + let auth_header = match credential_placement { + CredentialPlacement::Bearer => { ("authorization".to_string(), format!("Bearer {api_key}")) } - MessagesAuthStrategy::Header(name) => (name.to_string(), api_key), + CredentialPlacement::Header(name) => (name.to_string(), api_key), }; headers.push(auth_header); } diff --git a/litellm-rust/crates/core/src/messages/transformation.rs b/litellm-rust/crates/core/src/messages/transformation.rs index a5904c085a0..4957702987a 100644 --- a/litellm-rust/crates/core/src/messages/transformation.rs +++ b/litellm-rust/crates/core/src/messages/transformation.rs @@ -1,37 +1,17 @@ use super::types::{AnthropicMessagesRequest, AnthropicMessagesResponse}; use crate::Error; -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub enum MessagesAuthStrategy { - Bearer, - Header(&'static str), -} - -impl MessagesAuthStrategy { - pub fn header_name(self) -> &'static str { - match self { - Self::Bearer => "authorization", - Self::Header(header_name) => header_name, - } - } -} +use crate::auth::CredentialPlacement; pub trait AnthropicMessagesProviderConfig: Sync { - fn complete_url( - &self, - api_base: Option<&str>, - model: &str, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result; - fn resolve_api_key( &self, api_key: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, ) -> Result; - fn auth_strategy(&self) -> MessagesAuthStrategy { - MessagesAuthStrategy::Header("x-api-key") + fn credential_placement(&self) -> CredentialPlacement { + CredentialPlacement::Header("x-api-key") } fn accepts_bearer_auth(&self) -> bool { @@ -45,6 +25,13 @@ pub trait AnthropicMessagesProviderConfig: Sync { ] } + fn complete_url( + &self, + api_base: Option<&str>, + model: &str, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result; + #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn transform_request( &self, diff --git a/litellm-rust/crates/core/src/ocr/backends/azure_ai.rs b/litellm-rust/crates/core/src/ocr/backends/azure_ai.rs index 14f8c29df21..e8895b1cf10 100644 --- a/litellm-rust/crates/core/src/ocr/backends/azure_ai.rs +++ b/litellm-rust/crates/core/src/ocr/backends/azure_ai.rs @@ -1,6 +1,6 @@ use crate::auth::azure::AzureAuthInputs; use crate::constants::{AZURE_AI_OCR_PATH, AZURE_DI_API_VERSION}; -use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend}; +use crate::ocr::backends::{BackendConfig, OcrBackend, OcrIntegration, PreparedOcrBackend}; use crate::ocr::error::{OcrError, OcrRequestError}; use crate::ocr::formats::document_intelligence::{ AzureDocumentIntelligenceOcrFormat, @@ -29,7 +29,7 @@ fn encode_model_id(model: &str) -> Result { pub struct AzureMistral; impl OcrIntegration for AzureMistral { - type Host = AzureHost; + type Backend = AzureBackend; type Format = MistralOcrFormat; type PreparedDocument = crate::ocr::document::InlineOcrDocument; const FORMAT: Self::Format = MistralOcrFormat; @@ -38,7 +38,7 @@ impl OcrIntegration for AzureMistral { async fn prepare( &self, connection: &OcrConnection, - config: &HostConfig, + config: &BackendConfig, _model: &str, _params: &MistralOcrParams, env_lookup: &(dyn Fn(&str) -> Option + Sync), @@ -85,7 +85,7 @@ impl OcrIntegration for AzureMistral { pub struct AzureDocumentIntelligence; impl OcrIntegration for AzureDocumentIntelligence { - type Host = AzureHost; + type Backend = AzureBackend; type Format = AzureDocumentIntelligenceOcrFormat; type PreparedDocument = OcrDocument; const FORMAT: Self::Format = AzureDocumentIntelligenceOcrFormat; @@ -93,7 +93,7 @@ impl OcrIntegration for AzureDocumentIntelligence { async fn prepare( &self, connection: &OcrConnection, - config: &HostConfig, + config: &BackendConfig, model: &str, params: &DocumentIntelligenceParams, env_lookup: &(dyn Fn(&str) -> Option + Sync), @@ -166,9 +166,9 @@ impl OcrIntegration for AzureDocumentIntelligence { } #[derive(Clone, Debug)] -pub struct AzureHost; +pub struct AzureBackend; -impl OcrHost for AzureHost { +impl OcrBackend for AzureBackend { type Config = AzureAuthInputs; const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::AzureAi; } diff --git a/litellm-rust/crates/core/src/ocr/backends/mistral.rs b/litellm-rust/crates/core/src/ocr/backends/mistral.rs index 2266c9bb7c7..f72dc8a114e 100644 --- a/litellm-rust/crates/core/src/ocr/backends/mistral.rs +++ b/litellm-rust/crates/core/src/ocr/backends/mistral.rs @@ -1,5 +1,5 @@ use crate::constants::MISTRAL_OCR_API_BASE; -use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend}; +use crate::ocr::backends::{BackendConfig, OcrBackend, OcrIntegration, PreparedOcrBackend}; use crate::ocr::error::OcrError; use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams}; use crate::ocr::types::{OcrConnection, OcrDocument}; @@ -22,7 +22,7 @@ pub fn complete_url(api_base: Option<&str>) -> String { pub struct MistralDirect; impl OcrIntegration for MistralDirect { - type Host = MistralHost; + type Backend = MistralBackend; type Format = MistralOcrFormat; type PreparedDocument = OcrDocument; const FORMAT: Self::Format = MistralOcrFormat; @@ -30,7 +30,7 @@ impl OcrIntegration for MistralDirect { async fn prepare( &self, connection: &OcrConnection, - _config: &HostConfig, + _config: &BackendConfig, _model: &str, _params: &MistralOcrParams, env_lookup: &(dyn Fn(&str) -> Option + Sync), @@ -58,9 +58,9 @@ impl OcrIntegration for MistralDirect { } #[derive(Clone, Debug)] -pub struct MistralHost; +pub struct MistralBackend; -impl OcrHost for MistralHost { +impl OcrBackend for MistralBackend { type Config = (); const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::Mistral; } diff --git a/litellm-rust/crates/core/src/ocr/backends/mod.rs b/litellm-rust/crates/core/src/ocr/backends/mod.rs index d8c518137d3..f52202e1f8b 100644 --- a/litellm-rust/crates/core/src/ocr/backends/mod.rs +++ b/litellm-rust/crates/core/src/ocr/backends/mod.rs @@ -18,17 +18,17 @@ pub struct PreparedOcrBackend { } pub trait OcrIntegration: Send + Sync + Sized + 'static { - type Host: OcrHost; + type Backend: OcrBackend; type Format: OcrFormat; type PreparedDocument: Into<::PreparedDocument> + Send; const FORMAT: Self::Format; - fn provider_name(&self) -> &'static str { - Self::Host::PROVIDER.as_str() + fn supports_native_request_format(&self) -> bool { + false } - fn format(&self) -> Self::Format { - Self::FORMAT + fn guard_document_before_preparation(&self) -> bool { + false } fn decode_input_params( @@ -39,7 +39,7 @@ pub trait OcrIntegration: Send + Sync + Sized + 'static { super::registry::validate_request_format( ¶ms, self.supports_native_request_format(), - self.provider_name(), + Self::Backend::PROVIDER.as_str(), )?; Self::Format::validate_input_params(¶ms)?; super::wire::decode_request_value(serde_json::Value::Object(params), prefix) @@ -48,7 +48,7 @@ pub trait OcrIntegration: Send + Sync + Sized + 'static { fn prepare( &self, connection: &OcrConnection, - config: &HostConfig, + config: &BackendConfig, model: &str, params: &MappedParams, env_lookup: &(dyn Fn(&str) -> Option + Sync), @@ -73,10 +73,6 @@ pub trait OcrIntegration: Send + Sync + Sized + 'static { false } - fn guard_document_before_preparation(&self) -> bool { - false - } - fn read_response( &self, _client: &OcrClient, @@ -90,17 +86,13 @@ pub trait OcrIntegration: Send + Sync + Sized + 'static { > + Send { super::client::read_json_response(response, self.preserve_native_response(params)) } - - fn supports_native_request_format(&self) -> bool { - false - } } -pub trait OcrHost: Send + Sync + 'static { +pub trait OcrBackend: Send + Sync + 'static { type Config: Clone + std::fmt::Debug + Send + Sync + 'static; const PROVIDER: super::registry::OcrProvider; } pub type InputParams = <::Format as OcrFormat>::InputParams; pub type MappedParams = <::Format as OcrFormat>::MappedParams; -pub type HostConfig = <::Host as OcrHost>::Config; +pub type BackendConfig = <::Backend as OcrBackend>::Config; diff --git a/litellm-rust/crates/core/src/ocr/backends/reducto.rs b/litellm-rust/crates/core/src/ocr/backends/reducto.rs index b627f476366..5f31218e040 100644 --- a/litellm-rust/crates/core/src/ocr/backends/reducto.rs +++ b/litellm-rust/crates/core/src/ocr/backends/reducto.rs @@ -1,5 +1,5 @@ use crate::constants::{REDUCTO_ID_PREFIX, REDUCTO_OCR_API_BASE}; -use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend}; +use crate::ocr::backends::{BackendConfig, OcrBackend, OcrIntegration, PreparedOcrBackend}; use crate::ocr::document::InlineDocument; use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError}; use crate::ocr::formats::reducto::{ @@ -10,9 +10,9 @@ use crate::ocr::types::{OcrConnection, OcrDocument}; use crate::providers::reducto::auth; #[derive(Clone, Debug)] -pub struct ReductoHost; +pub struct ReductoBackend; -impl OcrHost for ReductoHost { +impl OcrBackend for ReductoBackend { type Config = (); const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::Reducto; } @@ -78,7 +78,7 @@ macro_rules! impl_reducto_backend { pub struct $integration; impl OcrIntegration for $integration { - type Host = ReductoHost; + type Backend = ReductoBackend; type Format = $format; type PreparedDocument = ReductoFileId; const FORMAT: Self::Format = $format; @@ -86,7 +86,7 @@ macro_rules! impl_reducto_backend { async fn prepare( &self, connection: &OcrConnection, - _config: &HostConfig, + _config: &BackendConfig, _model: &str, _params: &$params, env_lookup: &(dyn Fn(&str) -> Option + Sync), diff --git a/litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs b/litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs index daaeb9c5710..0289b4cac8f 100644 --- a/litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs +++ b/litellm-rust/crates/core/src/ocr/backends/vertex_ai.rs @@ -1,5 +1,5 @@ use crate::constants::{VERTEX_DEEPSEEK_API_BASE, VERTEX_OCR_DEFAULT_LOCATION}; -use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend}; +use crate::ocr::backends::{BackendConfig, OcrBackend, OcrIntegration, PreparedOcrBackend}; use crate::ocr::error::OcrError; use crate::ocr::formats::deepseek::{DeepSeekOcrFormat, types::DeepSeekOcrParams}; use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams}; @@ -30,7 +30,7 @@ fn location(config: &VertexAuthInputs, env_lookup: &dyn Fn(&str) -> Option, + config: &BackendConfig, model: &str, _params: &MistralOcrParams, env_lookup: &(dyn Fn(&str) -> Option + Sync), @@ -89,7 +89,7 @@ impl OcrIntegration for VertexMistral { pub struct VertexDeepSeek; impl OcrIntegration for VertexDeepSeek { - type Host = VertexHost; + type Backend = VertexBackend; type Format = DeepSeekOcrFormat; type PreparedDocument = OcrDocument; const FORMAT: Self::Format = DeepSeekOcrFormat; @@ -97,7 +97,7 @@ impl OcrIntegration for VertexDeepSeek { async fn prepare( &self, connection: &OcrConnection, - config: &HostConfig, + config: &BackendConfig, _model: &str, _params: &DeepSeekOcrParams, env_lookup: &(dyn Fn(&str) -> Option + Sync), @@ -130,9 +130,9 @@ impl OcrIntegration for VertexDeepSeek { } #[derive(Clone, Debug)] -pub struct VertexHost; +pub struct VertexBackend; -impl OcrHost for VertexHost { +impl OcrBackend for VertexBackend { type Config = VertexAuthInputs; const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::VertexAi; } diff --git a/litellm-rust/crates/core/src/ocr/formats/deepseek/mod.rs b/litellm-rust/crates/core/src/ocr/formats/deepseek/mod.rs index e8c50694c33..a052add8656 100644 --- a/litellm-rust/crates/core/src/ocr/formats/deepseek/mod.rs +++ b/litellm-rust/crates/core/src/ocr/formats/deepseek/mod.rs @@ -28,15 +28,12 @@ impl OcrFormat for DeepSeekOcrFormat { type RequestBody = DeepSeekOcrRequest; type ResponseBody = DeepSeekOcrResponse; #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn map_ocr_params( - &self, - params: Self::InputParams, - ) -> Result { + fn map_params(&self, params: Self::InputParams) -> Result { Ok(params) } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( + fn transform_request( &self, model: &str, document: OcrDocument, @@ -61,7 +58,7 @@ impl OcrFormat for DeepSeekOcrFormat { } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_response( + fn transform_response( &self, model: &str, response: Self::ResponseBody, diff --git a/litellm-rust/crates/core/src/ocr/formats/document_intelligence/mod.rs b/litellm-rust/crates/core/src/ocr/formats/document_intelligence/mod.rs index 9cbad1ac080..995ca5e9131 100644 --- a/litellm-rust/crates/core/src/ocr/formats/document_intelligence/mod.rs +++ b/litellm-rust/crates/core/src/ocr/formats/document_intelligence/mod.rs @@ -173,10 +173,7 @@ impl OcrFormat for AzureDocumentIntelligenceOcrFormat { } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn map_ocr_params( - &self, - params: Self::InputParams, - ) -> Result { + fn map_params(&self, params: Self::InputParams) -> Result { Ok(DocumentIntelligenceParams { pages: params .pages @@ -193,7 +190,7 @@ impl OcrFormat for AzureDocumentIntelligenceOcrFormat { } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( + fn transform_request( &self, _model: &str, document: OcrDocument, @@ -213,7 +210,7 @@ impl OcrFormat for AzureDocumentIntelligenceOcrFormat { } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_response( + fn transform_response( &self, model: &str, response: Self::ResponseBody, diff --git a/litellm-rust/crates/core/src/ocr/formats/document_intelligence/types.rs b/litellm-rust/crates/core/src/ocr/formats/document_intelligence/types.rs index 2f4806d8065..06e2f7a22e4 100644 --- a/litellm-rust/crates/core/src/ocr/formats/document_intelligence/types.rs +++ b/litellm-rust/crates/core/src/ocr/formats/document_intelligence/types.rs @@ -42,18 +42,6 @@ pub struct DocumentIntelligenceParams { pub(crate) request_format: OcrRequestFormat, } -impl From for DocumentIntelligenceInputParams { - fn from(params: DocumentIntelligenceParams) -> Self { - Self { - pages: params.pages.map(|pages| PagesInput::NativeRange(pages.0)), - features: params - .features - .map(|features| FeaturesInput::CommaSeparated(features.0)), - req_format: Some(params.request_format), - } - } -} - #[derive(Clone, Debug, Serialize, Deserialize)] pub enum DocumentIntelligenceRequest { #[serde(rename = "urlSource")] diff --git a/litellm-rust/crates/core/src/ocr/formats/mistral/mod.rs b/litellm-rust/crates/core/src/ocr/formats/mistral/mod.rs index fe5a3d96b08..2f6d8d05749 100644 --- a/litellm-rust/crates/core/src/ocr/formats/mistral/mod.rs +++ b/litellm-rust/crates/core/src/ocr/formats/mistral/mod.rs @@ -16,15 +16,12 @@ impl OcrFormat for MistralOcrFormat { type ResponseBody = MistralOcrResponse; #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn map_ocr_params( - &self, - params: MistralOcrParams, - ) -> Result { + fn map_params(&self, params: MistralOcrParams) -> Result { Ok(params) } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( + fn transform_request( &self, model: &str, document: OcrDocument, @@ -38,7 +35,7 @@ impl OcrFormat for MistralOcrFormat { } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_response( + fn transform_response( &self, model: &str, response: MistralOcrResponse, diff --git a/litellm-rust/crates/core/src/ocr/formats/mod.rs b/litellm-rust/crates/core/src/ocr/formats/mod.rs index 8ca047c53f7..56a2ce28f8e 100644 --- a/litellm-rust/crates/core/src/ocr/formats/mod.rs +++ b/litellm-rust/crates/core/src/ocr/formats/mod.rs @@ -13,7 +13,7 @@ pub mod reducto; pub trait OcrFormat: Send + Sync + Sized + 'static { type InputParams: std::fmt::Debug + Clone + Serialize + DeserializeOwned + Send + Sync; - type MappedParams: Clone + Serialize + Send + Sync + Into; + type MappedParams: Clone + Serialize + Send + Sync; type PreparedDocument: Send; type RequestBody: Serialize + DeserializeOwned + Send + Sync; type ResponseBody: DeserializeOwned + Send; @@ -22,17 +22,14 @@ pub trait OcrFormat: Send + Sync + Sized + 'static { Ok(()) } - fn map_ocr_params( - &self, - params: Self::InputParams, - ) -> Result; - fn transform_ocr_request( + fn map_params(&self, params: Self::InputParams) -> Result; + fn transform_request( &self, model: &str, document: Self::PreparedDocument, params: &Self::MappedParams, ) -> Result; - fn transform_ocr_response( + fn transform_response( &self, model: &str, response: Self::ResponseBody, diff --git a/litellm-rust/crates/core/src/ocr/formats/reducto/mod.rs b/litellm-rust/crates/core/src/ocr/formats/reducto/mod.rs index 9ba9978e908..bc6cd9c53e4 100644 --- a/litellm-rust/crates/core/src/ocr/formats/reducto/mod.rs +++ b/litellm-rust/crates/core/src/ocr/formats/reducto/mod.rs @@ -84,15 +84,12 @@ impl OcrFormat for ReductoParseV3Format { type ResponseBody = ReductoResponse; #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn map_ocr_params( - &self, - params: Self::InputParams, - ) -> Result { + fn map_params(&self, params: Self::InputParams) -> Result { Ok(params) } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( + fn transform_request( &self, _model: &str, document: ReductoFileId, @@ -105,7 +102,7 @@ impl OcrFormat for ReductoParseV3Format { } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_response( + fn transform_response( &self, model: &str, response: Self::ResponseBody, @@ -125,15 +122,12 @@ impl OcrFormat for ReductoParseLegacyFormat { type ResponseBody = ReductoResponse; #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn map_ocr_params( - &self, - params: Self::InputParams, - ) -> Result { + fn map_params(&self, params: Self::InputParams) -> Result { Ok(params) } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_request( + fn transform_request( &self, _model: &str, document: ReductoFileId, @@ -146,7 +140,7 @@ impl OcrFormat for ReductoParseLegacyFormat { } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_ocr_response( + fn transform_response( &self, model: &str, response: Self::ResponseBody, diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/core/src/ocr/handler.rs index 62ebf53a16b..22992ac33c1 100644 --- a/litellm-rust/crates/core/src/ocr/handler.rs +++ b/litellm-rust/crates/core/src/ocr/handler.rs @@ -109,7 +109,7 @@ where I: OcrIntegration, { let backend = &request.integration; - let format = &request.integration.format(); + let format = &I::FORMAT; let prepared_backend = backend .prepare( &request.connection, @@ -138,7 +138,7 @@ where let document = backend .prepare_document(client, document, &request.connection, &headers) .await?; - let body = format.transform_ocr_request(&request.model, document.into(), &request.params)?; + let body = format.transform_request(&request.model, document.into(), &request.params)?; let body = if !backend.guard_document_before_preparation() && hooks.has_guardrails() { let guarded = hooks .during_call(OcrDuringCallRequest { @@ -220,7 +220,7 @@ async fn send_ocr_call( body, } = request; let backend = &integration; - let format = integration.format(); + let format = I::FORMAT; let mut builder = client .provider_http() .post(&url) @@ -235,7 +235,7 @@ async fn send_ocr_call( let decoded = backend .read_response(client, response, &url, &headers, &connection, ¶ms) .await?; - let response = format.transform_ocr_response(&model, decoded.data, ¶ms)?; + let response = format.transform_response(&model, decoded.data, ¶ms)?; Ok(OcrResponseData { provider_native_response: decoded.native, ..response diff --git a/litellm-rust/crates/core/src/ocr/hooks.rs b/litellm-rust/crates/core/src/ocr/hooks.rs index 895ed54acfc..6ceacca7d79 100644 --- a/litellm-rust/crates/core/src/ocr/hooks.rs +++ b/litellm-rust/crates/core/src/ocr/hooks.rs @@ -118,7 +118,7 @@ where })?, "guardrail.optional_params", ) - .and_then(|params| request.integration.format().map_ocr_params(params))?; + .and_then(|params| I::FORMAT.map_params(params))?; Ok(Ok(MappedOcrRequest { document: changed.document, params, diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index c52e52365cf..2fc885f61d9 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -1,4 +1,4 @@ -use super::backends::{HostConfig, InputParams, MappedParams, OcrIntegration}; +use super::backends::{BackendConfig, InputParams, MappedParams, OcrIntegration}; use super::formats::OcrFormat; use super::types::{OcrConnection, OcrDocument}; @@ -16,7 +16,7 @@ where pub model: String, pub document: OcrDocument, pub params: MappedParams, - pub backend_config: HostConfig, + pub backend_config: BackendConfig, pub connection: OcrConnection, } @@ -26,13 +26,13 @@ pub(crate) fn prepare_ocr_call( model: String, document: OcrDocument, params: InputParams, - backend_config: HostConfig, + backend_config: BackendConfig, connection: OcrConnection, ) -> Result, Error> where I: OcrIntegration, { - let params = integration.format().map_ocr_params(params)?; + let params = I::FORMAT.map_params(params)?; Ok(MappedOcrRequest { integration, model, diff --git a/litellm-rust/crates/core/src/ocr/registry.rs b/litellm-rust/crates/core/src/ocr/registry.rs index 95c38b2b91b..4c226588c10 100644 --- a/litellm-rust/crates/core/src/ocr/registry.rs +++ b/litellm-rust/crates/core/src/ocr/registry.rs @@ -10,8 +10,8 @@ pub use super::backends::azure_ai::{AzureDocumentIntelligence, AzureMistral}; pub use super::backends::mistral::MistralDirect; pub use super::backends::reducto::{ReductoLegacy, ReductoV3}; pub use super::backends::vertex_ai::{VertexDeepSeek, VertexMistral}; -use super::backends::{HostConfig, InputParams}; -pub use super::backends::{OcrHost, OcrIntegration}; +use super::backends::{BackendConfig, InputParams}; +pub use super::backends::{OcrBackend, OcrIntegration}; use super::types::OcrRequestFormat; pub(crate) fn validate_request_format( @@ -67,7 +67,7 @@ pub(crate) const REDUCTO_LEGACY: ReductoLegacy = ReductoLegacy; #[derive(Clone, Debug)] pub struct OcrIntegrationInput { pub params: InputParams, - pub backend_config: HostConfig, + pub backend_config: BackendConfig, } #[derive(Clone, Debug)] diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs index 22a50c62c1e..43652fd9c70 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs @@ -1,14 +1,14 @@ +use crate::auth::RequestAuth; use crate::chat_completions::error::{ChatRequestError, ChatResponseError}; -use serde_json::{Map, Value, json}; +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; use crate::chat_completions::conversation::{Conversation, build_conversation}; use crate::chat_completions::transformation::{ - ChatCompletionsAuth, ChatCompletionsProviderConfig, Unsupported, unsupported_message, - unsupported_param, + ChatCompletionsProviderConfig, Unsupported, unsupported_message, unsupported_param, }; use crate::chat_completions::types::{ ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, ChatMessage, - ProviderChatRequestData, ProviderChatResponseData, }; use crate::constants::ANTHROPIC_OAUTH_TOKEN_PREFIX; use crate::error::Error; @@ -28,52 +28,100 @@ use crate::chat_completions::response_utils::{finish_reason_for, unix_now, usage /// per-model gate inside `transform_request`, the function this route replaces. /// Forwarding it would send `top_k` to a model that removed sampling params and /// take a 400 after the call, where Python drops it and succeeds. -const SUPPORTED_PARAMS: &[(&str, &str)] = &[ - ("max_tokens", "max_tokens"), - ("temperature", "temperature"), - ("top_p", "top_p"), - ("stop", "stop_sequences"), -]; +const SUPPORTED_PARAMS: &[&str] = &["max_tokens", "temperature", "top_p", "stop_sequences"]; pub struct AnthropicChatCompletionsConfig; pub const ANTHROPIC_CHAT_COMPLETIONS_CONFIG: AnthropicChatCompletionsConfig = AnthropicChatCompletionsConfig; -fn text_block(text: &str) -> Value { - json!({"type": "text", "text": text}) +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct AnthropicMappedParams { + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::http_utils::deserialize_optional_param" + )] + pub max_tokens: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::http_utils::deserialize_optional_param" + )] + pub temperature: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::http_utils::deserialize_optional_param" + )] + pub top_p: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::http_utils::deserialize_optional_param" + )] + pub stop_sequences: Option>>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::http_utils::deserialize_optional_param" + )] + pub stream: Option>, } -fn anthropic_body(model: &str, conversation: &Conversation, params: Map) -> Value { - let messages: Vec = conversation - .turns - .iter() - .map(|turn| { - json!({ - "role": turn.role.as_str(), - "content": turn.texts.iter().map(|text| text_block(text)).collect::>(), - }) - }) - .collect(); +#[derive(Clone, Debug, Serialize)] +pub struct AnthropicChatRequest { + pub model: String, + pub messages: Vec, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub system: Vec, + #[serde(flatten)] + pub params: AnthropicMappedParams, +} - let system: Vec = conversation.system.iter().map(|s| text_block(s)).collect(); +#[derive(Clone, Debug, Serialize)] +pub struct AnthropicTextMessage { + pub role: crate::chat_completions::conversation::TurnRole, + pub content: Vec, +} - let body = Map::from_iter( - [ - ("model".to_string(), json!(model)), - ("messages".to_string(), json!(messages)), - ] - .into_iter() - // Python builds `{"model", "messages", **optional_params}` with - // `system` already folded into optional_params, so a caller-supplied - // key of the same name wins here too. - .chain((!system.is_empty()).then(|| ("system".to_string(), json!(system)))) - .chain(params), - ); - Value::Object(body) +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AnthropicTextBlock { + Text { text: String }, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct AnthropicChatResponse { + pub model: Option, + pub content: Option>, + pub stop_reason: Option, + pub usage: Option, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AnthropicResponseBlock { + Text { + text: Option, + }, + #[serde(other)] + Other, +} + +#[derive(Clone, Debug, Default, Deserialize)] +pub struct AnthropicChatUsage { + pub input_tokens: Option, + pub output_tokens: Option, + pub cache_read_input_tokens: Option, + pub cache_creation_input_tokens: Option, } impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { + type MappedParams = AnthropicMappedParams; + type RequestBody = AnthropicChatRequest; + type ResponseBody = AnthropicChatResponse; + fn complete_url( &self, api_base: Option<&str>, @@ -90,8 +138,8 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { _model: &str, _optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - Ok(ChatCompletionsAuth::Header { + ) -> Result { + Ok(RequestAuth::Header { name: "x-api-key", value: resolve_anthropic_api_key(api_key, env_lookup)?, }) @@ -119,16 +167,16 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)] { + fn supported_provider_params(&self) -> &'static [&'static str] { SUPPORTED_PARAMS } - fn unsupported_reason( + fn decline_reason( &self, messages: &[ChatMessage], optional_params: &Map, ) -> Option { - unsupported_param(self.supported_openai_params(), &[], optional_params) + unsupported_param(self.supported_provider_params(), &[], optional_params) .or_else(|| messages.iter().find_map(unsupported_message)) // Anthropic rejects a request whose first turn is not a user turn. // Python only repairs that under `litellm.modify_params`, which the @@ -143,11 +191,29 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { fn transform_request( &self, model: &str, - messages: Vec, - optional_params: Map, - ) -> Result { - Ok(ProviderChatRequestData { - body: anthropic_body(model, &build_conversation(&messages), optional_params), + conversation: Conversation, + params: Self::MappedParams, + ) -> Result { + Ok(AnthropicChatRequest { + model: model.to_string(), + messages: conversation + .turns + .into_iter() + .map(|turn| AnthropicTextMessage { + role: turn.role, + content: turn + .texts + .into_iter() + .map(|text| AnthropicTextBlock::Text { text }) + .collect(), + }) + .collect(), + system: conversation + .system + .into_iter() + .map(|text| AnthropicTextBlock::Text { text }) + .collect(), + params, }) } @@ -155,59 +221,41 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { fn transform_response( &self, _model: &str, - response: ProviderChatResponseData, + response: Self::ResponseBody, ) -> Result { - let body = response - .body - .as_object() - .ok_or(ChatResponseError::NotObject { api: "messages" })?; - - let content = body - .get("content") - .and_then(Value::as_array) + let content = response + .content .ok_or(ChatResponseError::MissingField("content"))?; - if content - .iter() - .any(|block| block.get("type").and_then(Value::as_str) != Some("text")) - { - return Err(ChatResponseError::NonTextContent); - } - let text: String = content - .iter() - .filter_map(|block| block.get("text").and_then(Value::as_str)) - .collect(); - - let usage = body - .get("usage") - .and_then(Value::as_object) + let text = content + .into_iter() + .map(|block| match block { + AnthropicResponseBlock::Text { text } => Ok(text.unwrap_or_default()), + AnthropicResponseBlock::Other => Err(ChatResponseError::NonTextContent), + }) + .collect::>()?; + let usage = response + .usage .ok_or(ChatResponseError::MissingField("usage"))?; - let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0); Ok(ChatCompletionsResponse { created: unix_now(), - model: body - .get("model") - .and_then(Value::as_str) - .ok_or(ChatResponseError::MissingField("model"))? - .to_string(), + model: response + .model + .ok_or(ChatResponseError::MissingField("model"))?, choices: vec![ChatCompletionsChoice { index: 0, message: ChatCompletionsChoiceMessage { role: "assistant".to_string(), content: (!text.is_empty()).then_some(text), }, - finish_reason: finish_reason_for( - body.get("stop_reason") - .and_then(Value::as_str) - .unwrap_or(""), - ) - .to_string(), + finish_reason: finish_reason_for(response.stop_reason.as_deref().unwrap_or("")) + .to_string(), }], usage: usage_from_parts( - field("input_tokens"), - field("output_tokens"), - field("cache_read_input_tokens"), - field("cache_creation_input_tokens"), + usage.input_tokens.unwrap_or(0), + usage.output_tokens.unwrap_or(0), + usage.cache_read_input_tokens.unwrap_or(0), + usage.cache_creation_input_tokens.unwrap_or(0), ), }) } diff --git a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs index 4c9934c3e5f..df6a09d718f 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/messages/transformation.rs @@ -1,6 +1,7 @@ +use crate::auth::CredentialPlacement; use crate::auth::error::MissingCredential; use crate::error::{AuthError, Error}; -use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; +use crate::messages::transformation::AnthropicMessagesProviderConfig; const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; const ANTHROPIC_API_BASE_ENV: &str = "ANTHROPIC_API_BASE"; @@ -62,8 +63,8 @@ impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig { resolve_anthropic_api_key(api_key, env_lookup) } - fn auth_strategy(&self) -> MessagesAuthStrategy { - MessagesAuthStrategy::Header("x-api-key") + fn credential_placement(&self) -> CredentialPlacement { + CredentialPlacement::Header("x-api-key") } } @@ -126,7 +127,9 @@ mod tests { #[test] fn auth_strategy_and_default_headers_match_anthropic() { assert_eq!( - ANTHROPIC_MESSAGES_CONFIG.auth_strategy().header_name(), + ANTHROPIC_MESSAGES_CONFIG + .credential_placement() + .header_name(), "x-api-key" ); assert_eq!( diff --git a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs index 7cbd8d576e1..78d2a3c0bc2 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs @@ -1,6 +1,7 @@ +use crate::auth::CredentialPlacement; use crate::auth::error::MissingCredential; use crate::error::{AuthError, Error}; -use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy}; +use crate::messages::transformation::AnthropicMessagesProviderConfig; use crate::messages::types::{ AnthropicMessage, AnthropicMessagesRequest, AnthropicMessagesResponse, ContentBlock, MessageContent, SystemPrompt, @@ -154,8 +155,8 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig { resolve_azure_api_key(api_key, env_lookup) } - fn auth_strategy(&self) -> MessagesAuthStrategy { - self.anthropic.auth_strategy() + fn credential_placement(&self) -> CredentialPlacement { + self.anthropic.credential_placement() } fn accepts_bearer_auth(&self) -> bool { @@ -287,7 +288,7 @@ mod tests { fn auth_strategy_is_x_api_key() { assert_eq!( AZURE_ANTHROPIC_MESSAGES_CONFIG - .auth_strategy() + .credential_placement() .header_name(), "x-api-key" ); diff --git a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs index 9bf1f73a74d..f0d13d44ab2 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs @@ -1,110 +1,87 @@ -use serde_json::{Map, Value, json}; +use super::converse::{ + ConverseAudio, ConverseAudioSource, ConverseContent, ConverseMappedParams, ConverseMessage, + ConverseRequest, ConverseResponseContent, ConverseText, +}; +use crate::auth::RequestAuth; +use crate::chat_completions::conversation::TurnRole; +use serde_json::{Map, Value}; -use crate::audio_transcription::transformation::{ - AudioTranscriptionAuth, AudioTranscriptionProviderConfig, -}; +use crate::audio_transcription::transformation::AudioTranscriptionProviderConfig; use crate::audio_transcription::types::{ - AudioTranscriptionRequestData, AudioTranscriptionResponseData, + AudioTranscriptionRequestData, AudioTranscriptionResponseData, ProviderTranscriptionResponse, + TranscriptionAudio, TranscriptionParams, }; -use crate::error::{Error, json_type_name}; +use crate::error::Error; pub use super::aws_base::{aws_auth_config, bedrock_model_id_and_region, resolve_bedrock_region}; -use super::constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}; - -const SUPPORTED_PARAMS: &[&str] = &["language", "prompt", "temperature", "response_format"]; +use super::constants::BEDROCK_RUNTIME_ENDPOINT_TEMPLATE; pub static BEDROCK_AUDIO_TRANSCRIPTION_CONFIG: BedrockAudioTranscriptionConfig = BedrockAudioTranscriptionConfig; pub struct BedrockAudioTranscriptionConfig; -fn audio_fields(audio: Value) -> Result<(String, String), Error> { - let object = audio.as_object().ok_or_else(|| Error::InvalidType { - expected: "object", - actual: json_type_name(&audio), - })?; - let data = object - .get("data") - .and_then(Value::as_str) - .filter(|value| !value.is_empty()) - .ok_or(Error::MissingField("audio.data"))?; - let format = object - .get("format") - .and_then(Value::as_str) - .filter(|value| matches!(*value, "wav" | "mp3" | "flac" | "ogg")) - .ok_or_else(|| { - Error::InvalidRequest("audio.format must be wav, mp3, flac, or ogg".to_string()) - })?; - Ok((data.to_string(), format.to_string())) -} - -fn optional_string<'a>(params: &'a Map, key: &str) -> Option<&'a str> { - params - .get(key) - .and_then(Value::as_str) - .filter(|value| !value.is_empty()) -} - impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn supported_transcription_params(&self) -> &'static [&'static str] { - SUPPORTED_PARAMS - } - - #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_transcription_request( + fn transform_request( &self, _model: &str, - audio: Value, - optional_params: Map, + audio: TranscriptionAudio, + params: TranscriptionParams, ) -> Result { - let (data, format) = audio_fields(audio)?; + if audio.data.is_empty() { + return Err(Error::MissingField("audio.data")); + } let mut instruction = "Transcribe the audio. Respond with only the transcript.".to_string(); - if let Some(language) = optional_string(&optional_params, "language") { + if let Some(language) = params.language.as_deref().filter(|value| !value.is_empty()) { instruction.push_str(&format!(" The audio language is {language}.")); } - if let Some(prompt) = optional_string(&optional_params, "prompt") { + if let Some(prompt) = params.prompt.as_deref().filter(|value| !value.is_empty()) { instruction.push_str(&format!(" Additional context: {prompt}")); } - let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]); - if let Some(temperature) = optional_params.get("temperature") { - inference_config.insert("temperature".to_string(), temperature.clone()); - } - Ok(AudioTranscriptionRequestData { - body: json!({ - "messages": [{ - "role": "user", - "content": [ - {"audio": {"format": format, "source": {"bytes": data}}}, - {"text": instruction} - ] - }], - "system": [{"text": "You are a transcription assistant."}], - "inferenceConfig": inference_config, - }), - }) + Ok(AudioTranscriptionRequestData::Bedrock(ConverseRequest { + messages: vec![ConverseMessage { + role: TurnRole::User, + content: vec![ + ConverseContent::Audio(ConverseAudio { + format: audio.format, + source: ConverseAudioSource { bytes: audio.data }, + }), + ConverseContent::Text(instruction), + ], + }], + system: vec![ConverseText { + text: "You are a transcription assistant.".into(), + }], + inference_config: ConverseMappedParams { + max_tokens: Some(Some(4096)), + temperature: params.temperature, + ..Default::default() + }, + })) } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn transform_transcription_response( + fn transform_response( &self, _model: &str, - response_json: Value, + response: ProviderTranscriptionResponse, ) -> Result { - let content = response_json - .get("output") - .and_then(|value| value.get("message")) - .and_then(|value| value.get("content")) - .and_then(Value::as_array) + let ProviderTranscriptionResponse::Bedrock(response) = response; + let content = response + .output + .and_then(|output| output.message) + .and_then(|message| message.content) .ok_or_else(|| { - Error::InvalidResponse("Bedrock response has no output content".to_string()) + Error::InvalidResponse("Bedrock response has no output content".into()) })?; - let mut text = String::new(); - for block in content { - if let Some(value) = block.get("text").and_then(Value::as_str) { - text.push_str(value); - } - } + let text = content + .into_iter() + .filter_map(|block| match block { + ConverseResponseContent::Object(block) => block.text.flatten(), + ConverseResponseContent::Other(_) => None, + }) + .collect(); Ok(AudioTranscriptionResponseData { text }) } @@ -131,16 +108,16 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { )) } - fn auth_strategy( + fn auth( &self, + _api_key: Option<&str>, model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { + ) -> Result { let (_, model_region) = bedrock_model_id_and_region(model); - Ok(AudioTranscriptionAuth::AwsSigV4 { + Ok(RequestAuth::AwsSigV4 { region: resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup), - service: BEDROCK_SERVICE, }) } } @@ -148,6 +125,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { #[cfg(test)] mod tests { use super::*; + use serde_json::json; fn no_env(_: &str) -> Option { None @@ -161,16 +139,19 @@ mod tests { ("temperature".to_string(), json!(0)), ("timestamp_granularities".to_string(), json!(["word"])), ]); - let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(¶ms); + let params = serde_json::from_value(Value::Object(params)).unwrap(); let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG - .transform_transcription_request( + .transform_request( "mistral.voxtral-mini-3b-2507", - json!({"data": "AQI=", "format": "wav", "filename": "sample.wav"}), + serde_json::from_value( + json!({"data": "AQI=", "format": "wav", "filename": "sample.wav"}), + ) + .unwrap(), params, ) .expect("request"); assert_eq!( - result.body, + serde_json::to_value(result).unwrap(), json!({ "messages": [{ "role": "user", @@ -188,9 +169,9 @@ mod tests { #[test] fn response_concatenates_content_blocks() { let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG - .transform_transcription_response( + .transform_response( "model", - json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}}), + serde_json::from_value(json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}})).unwrap(), ) .expect("response"); assert_eq!(result.text, "hello world"); @@ -199,10 +180,13 @@ mod tests { #[test] fn invalid_audio_is_rejected() { - let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.transform_transcription_request( + let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.transform_request( "model", - json!({"data": "AQI="}), - Map::new(), + TranscriptionAudio { + data: String::new(), + format: crate::audio_transcription::types::AudioFormat::Wav, + }, + TranscriptionParams::default(), ); assert!(result.is_err()); } diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs index cb28b8a4515..abcbf716994 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs @@ -1,16 +1,19 @@ +use crate::auth::RequestAuth; use crate::chat_completions::error::{ChatRequestError, ChatResponseError}; -use serde_json::{Map, Value, json}; +use crate::providers::bedrock::converse::{ + ConverseContent, ConverseMappedParams, ConverseMessage, ConverseRequest, ConverseResponse, + ConverseResponseContent, ConverseText, +}; +use serde_json::{Map, Value}; use crate::chat_completions::conversation::{Conversation, TurnRole, build_conversation}; use crate::chat_completions::response_utils::{finish_reason_for, unix_now, usage_from_parts}; use crate::chat_completions::transformation::{ - ChatCompletionsAuth, ChatCompletionsProviderConfig, Unsupported, unsupported_message, - unsupported_param, + ChatCompletionsProviderConfig, Unsupported, unsupported_message, unsupported_param, }; use crate::chat_completions::types::{ ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, - ChatCompletionsUsage, ChatMessage, ChatMessageContent, ProviderChatRequestData, - ProviderChatResponseData, + ChatCompletionsUsage, ChatMessage, ChatMessageContent, }; use crate::error::Error; @@ -24,12 +27,7 @@ use super::super::constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT /// `additionalModelRequestFields` for Anthropic base models and to /// `inferenceConfig` otherwise, and that branch reads the model catalog the /// core cannot see. -const SUPPORTED_PARAMS: &[(&str, &str)] = &[ - ("max_tokens", "maxTokens"), - ("temperature", "temperature"), - ("top_p", "topP"), - ("stop", "stopSequences"), -]; +const SUPPORTED_PARAMS: &[&str] = &["maxTokens", "temperature", "topP", "stopSequences"]; const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = "aws_bedrock_runtime_endpoint"; @@ -56,43 +54,6 @@ pub struct BedrockChatCompletionsConfig; pub const BEDROCK_CHAT_COMPLETIONS_CONFIG: BedrockChatCompletionsConfig = BedrockChatCompletionsConfig; -fn converse_body(conversation: &Conversation, params: &Map) -> Value { - let messages: Vec = conversation - .turns - .iter() - .map(|turn| { - json!({ - "role": turn.role.as_str(), - "content": turn.texts.iter().map(|text| json!({"text": text})).collect::>(), - }) - }) - .collect(); - - let inference_config = Map::from_iter(SUPPORTED_PARAMS.iter().filter_map(|(_, name)| { - params - .get(*name) - .map(|value| ((*name).to_string(), value.clone())) - })); - - let system: Vec = conversation - .system - .iter() - .map(|text| json!({"text": text})) - .collect(); - - Value::Object(Map::from_iter( - [ - ( - "inferenceConfig".to_string(), - Value::Object(inference_config), - ), - ("messages".to_string(), json!(messages)), - ] - .into_iter() - .chain((!system.is_empty()).then(|| ("system".to_string(), json!(system)))), - )) -} - fn has_blank_text(message: &ChatMessage) -> bool { match &message.content { None => false, @@ -106,6 +67,10 @@ fn has_blank_text(message: &ChatMessage) -> bool { } impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { + type MappedParams = ConverseMappedParams; + type RequestBody = ConverseRequest; + type ResponseBody = ConverseResponse; + fn complete_url( &self, api_base: Option<&str>, @@ -139,7 +104,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { model: &str, optional_params: &Map, env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { + ) -> Result { // Python reads `api_key` as the Bedrock bearer token and consults the // env only when the caller passed none, so a caller-supplied empty key // falls through to SigV4 without reaching for the environment. An @@ -152,10 +117,10 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { } .filter(|token| !token.is_empty()); if let Some(token) = bearer { - return Ok(ChatCompletionsAuth::Bearer { token }); + return Ok(RequestAuth::Bearer { token }); } let (_, model_region) = bedrock_model_id_and_region(model); - Ok(ChatCompletionsAuth::AwsSigV4 { + Ok(RequestAuth::AwsSigV4 { region: resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup), }) } @@ -165,7 +130,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { } #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] - fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)] { + fn supported_provider_params(&self) -> &'static [&'static str] { SUPPORTED_PARAMS } @@ -173,13 +138,13 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { CONFIG_PARAMS } - fn unsupported_reason( + fn decline_reason( &self, messages: &[ChatMessage], optional_params: &Map, ) -> Option { unsupported_param( - self.supported_openai_params(), + self.supported_provider_params(), CONFIG_PARAMS, optional_params, ) @@ -213,52 +178,56 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { fn transform_request( &self, _model: &str, - messages: Vec, - optional_params: Map, - ) -> Result { - Ok(ProviderChatRequestData { - body: converse_body(&build_conversation(&messages), &optional_params), + conversation: Conversation, + params: Self::MappedParams, + ) -> Result { + Ok(ConverseRequest { + messages: conversation + .turns + .into_iter() + .map(|turn| ConverseMessage { + role: turn.role, + content: turn.texts.into_iter().map(ConverseContent::Text).collect(), + }) + .collect(), + system: conversation + .system + .into_iter() + .map(|text| ConverseText { text }) + .collect(), + inference_config: params, }) } fn transform_response( &self, model: &str, - response: ProviderChatResponseData, + response: Self::ResponseBody, ) -> Result { - let body = response - .body - .as_object() - .ok_or(ChatResponseError::NotObject { api: "converse" })?; - - let content = body - .get("output") - .and_then(|output| output.get("message")) - .and_then(|message| message.get("content")) - .and_then(Value::as_array) + let content = response + .output + .and_then(|output| output.message) + .and_then(|message| message.content) .ok_or(ChatResponseError::MissingField("output.message.content"))?; - if content.iter().any(|block| { - block - .as_object() - .is_none_or(|block| block.len() != 1 || !block.contains_key("text")) - }) { - return Err(ChatResponseError::NonTextContent); - } - let text: String = content - .iter() - .filter_map(|block| block.get("text").and_then(Value::as_str)) - .collect(); - - let usage = body - .get("usage") - .and_then(Value::as_object) + let text = content + .into_iter() + .map(|block| match block { + ConverseResponseContent::Object(block) + if block.text.is_some() && block.other_fields.is_empty() => + { + Ok(block.text.flatten().unwrap_or_default()) + } + _ => Err(ChatResponseError::NonTextContent), + }) + .collect::>()?; + let usage = response + .usage .ok_or(ChatResponseError::MissingField("usage"))?; - let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0); let computed = usage_from_parts( - field("inputTokens"), - field("outputTokens"), - field("cacheReadInputTokens"), - field("cacheWriteInputTokens"), + usage.input_tokens.unwrap_or(0), + usage.output_tokens.unwrap_or(0), + usage.cache_read_input_tokens.unwrap_or(0), + usage.cache_write_input_tokens.unwrap_or(0), ); // Converse reports `totalTokens` and Python passes it straight through, // where Anthropic has no such field and Python adds the two counts @@ -267,10 +236,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { // raises there rather than reporting a zero; fall back to the computed // total, which is the closest thing to that without failing the call. let usage = ChatCompletionsUsage { - total_tokens: usage - .get("totalTokens") - .and_then(Value::as_u64) - .unwrap_or(computed.total_tokens), + total_tokens: usage.total_tokens.unwrap_or(computed.total_tokens), ..computed }; @@ -288,10 +254,8 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { // on this path alone. content: Some(text), }, - finish_reason: finish_reason_for( - body.get("stopReason").and_then(Value::as_str).unwrap_or(""), - ) - .to_string(), + finish_reason: finish_reason_for(response.stop_reason.as_deref().unwrap_or("")) + .to_string(), }], usage, }) diff --git a/litellm-rust/crates/core/src/providers/bedrock/converse.rs b/litellm-rust/crates/core/src/providers/bedrock/converse.rs new file mode 100644 index 00000000000..40a00359b62 --- /dev/null +++ b/litellm-rust/crates/core/src/providers/bedrock/converse.rs @@ -0,0 +1,116 @@ +use serde::{Deserialize, Serialize}; + +use crate::chat_completions::conversation::TurnRole; + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConverseMappedParams { + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::http_utils::deserialize_optional_param" + )] + pub max_tokens: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::http_utils::deserialize_optional_param" + )] + pub temperature: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::http_utils::deserialize_optional_param" + )] + pub top_p: Option>, + #[serde( + default, + skip_serializing_if = "Option::is_none", + deserialize_with = "crate::http_utils::deserialize_optional_param" + )] + pub stop_sequences: Option>>, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConverseRequest { + pub messages: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub system: Vec, + pub inference_config: ConverseMappedParams, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ConverseMessage { + pub role: TurnRole, + pub content: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum ConverseContent { + Text(String), + Audio(ConverseAudio), +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ConverseText { + pub text: String, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ConverseAudio { + pub format: crate::audio_transcription::types::AudioFormat, + pub source: ConverseAudioSource, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ConverseAudioSource { + pub bytes: String, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ConverseResponse { + pub output: Option, + #[serde(rename = "stopReason")] + pub stop_reason: Option, + pub usage: Option, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ConverseOutput { + pub message: Option, +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ConverseResponseMessage { + pub content: Option>, +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum ConverseResponseContent { + Object(ConverseResponseObject), + Other(serde::de::IgnoredAny), +} + +#[derive(Clone, Debug, Deserialize)] +pub struct ConverseResponseObject { + #[serde( + default, + deserialize_with = "crate::http_utils::deserialize_optional_param" + )] + pub text: Option>, + #[serde(flatten)] + pub other_fields: std::collections::BTreeMap, +} + +#[derive(Clone, Debug, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ConverseUsage { + pub input_tokens: Option, + pub output_tokens: Option, + pub total_tokens: Option, + pub cache_read_input_tokens: Option, + pub cache_write_input_tokens: Option, +} diff --git a/litellm-rust/crates/core/src/providers/bedrock/mod.rs b/litellm-rust/crates/core/src/providers/bedrock/mod.rs index 32934af7211..b8131447388 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/mod.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/mod.rs @@ -7,4 +7,5 @@ pub mod audio_transcription; pub mod aws_base; pub mod chat_completions; mod constants; +pub mod converse; mod resolved_credentials_cache; diff --git a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs index f1985f81b7d..48c353f9e4a 100644 --- a/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs +++ b/litellm-rust/crates/core/src/providers/openai/realtime/transformation.rs @@ -8,23 +8,6 @@ pub const OPENAI_REALTIME_DEFAULT_API_BASE: &str = "https://api.openai.com"; /// Path appended to the resolved host base to reach the realtime endpoint. pub const OPENAI_REALTIME_PATH: &str = "/v1/realtime"; -/// Percent-encode a query value, escaping any char outside the RFC 3986 -/// unreserved set (`A-Za-z0-9-._~`). Keeps us dependency-free; common realtime -/// model slugs have no special chars, but this stays correct for the rest. -fn percent_encode(value: &str) -> String { - let mut encoded = String::with_capacity(value.len()); - for byte in value.bytes() { - let unreserved = byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~'); - if unreserved { - encoded.push(byte as char); - } else { - encoded.push('%'); - encoded.push_str(&format!("{byte:02X}")); - } - } - encoded -} - /// Build the realtime WebSocket URL, porting Python's `OpenAIRealtime._construct_url`. /// /// Blank/whitespace `api_base` is treated as absent (guard at resolution time), @@ -55,7 +38,7 @@ pub fn complete_url(api_base: Option<&str>, model: &str) -> String { format!( "{base}{OPENAI_REALTIME_PATH}?model={}", - percent_encode(model) + crate::http_utils::encode_query_value(model) ) } @@ -68,7 +51,7 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { complete_url(api_base, model) } - fn transform_realtime_request( + fn transform_request( &self, event: &RealtimeEvent, _model: &str, @@ -76,7 +59,7 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { Ok(RealtimeTransformResult::passthrough(event.clone())) } - fn transform_realtime_response( + fn transform_response( &self, event: &RealtimeEvent, _model: &str, @@ -85,18 +68,18 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig { } } -pub fn transform_realtime_request( +pub fn transform_request( event: &RealtimeEvent, model: &str, ) -> Result { - OPENAI_REALTIME_CONFIG.transform_realtime_request(event, model) + OPENAI_REALTIME_CONFIG.transform_request(event, model) } -pub fn transform_realtime_response( +pub fn transform_response( event: &RealtimeEvent, model: &str, ) -> Result { - OPENAI_REALTIME_CONFIG.transform_realtime_response(event, model) + OPENAI_REALTIME_CONFIG.transform_response(event, model) } #[cfg(test)] @@ -172,8 +155,7 @@ mod tests { let event: RealtimeEvent = serde_json::from_str(r#"{"type":"session.update","session":{"voice":"alloy"}}"#) .expect("valid event"); - let result = - transform_realtime_request(&event, "gpt-realtime").expect("passthrough is infallible"); + let result = transform_request(&event, "gpt-realtime").expect("passthrough is infallible"); assert_eq!(result.events, vec![event]); } @@ -182,8 +164,7 @@ mod tests { let event: RealtimeEvent = serde_json::from_str(r#"{"type":"response.output_audio.delta","delta":"abc=="}"#) .expect("valid event"); - let result = - transform_realtime_response(&event, "gpt-realtime").expect("passthrough is infallible"); + let result = transform_response(&event, "gpt-realtime").expect("passthrough is infallible"); assert_eq!(result.events, vec![event]); } } diff --git a/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs b/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs index be86bb90311..14189d3212f 100644 --- a/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs +++ b/litellm-rust/crates/core/src/providers/openai/responses/transformation.rs @@ -2,16 +2,12 @@ use crate::Error; use crate::responses::types::{ResponsesWsEvent, ResponsesWsTransformResult}; use crate::responses::websocket::{ResponsesWebSocketProviderConfig, enforce_model}; -pub struct OpenAIResponsesWsConfig; +pub struct OpenAiResponsesWsConfig; -pub const OPENAI_RESPONSES_WS_CONFIG: OpenAIResponsesWsConfig = OpenAIResponsesWsConfig; +pub const OPENAI_RESPONSES_WS_CONFIG: OpenAiResponsesWsConfig = OpenAiResponsesWsConfig; -impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig { - fn supports_native_websocket(&self) -> bool { - true - } - - fn transform_ws_request( +impl ResponsesWebSocketProviderConfig for OpenAiResponsesWsConfig { + fn transform_request( &self, event: &ResponsesWsEvent, model: &str, @@ -21,7 +17,7 @@ impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig { ))) } - fn transform_ws_response( + fn transform_response( &self, event: &ResponsesWsEvent, _model: &str, @@ -40,9 +36,8 @@ mod tests { serde_json::from_value(serde_json::json!({"type":"response.create"})) .expect("valid event"); let result = OPENAI_RESPONSES_WS_CONFIG - .transform_ws_request(&event, "gpt-5") + .transform_request(&event, "gpt-5") .expect("valid transform"); assert_eq!(result.events[0].model(), Some("gpt-5")); - assert!(OPENAI_RESPONSES_WS_CONFIG.supports_native_websocket()); } } diff --git a/litellm-rust/crates/core/src/realtime/transformation.rs b/litellm-rust/crates/core/src/realtime/transformation.rs index b08084514ef..4b9f33586ad 100644 --- a/litellm-rust/crates/core/src/realtime/transformation.rs +++ b/litellm-rust/crates/core/src/realtime/transformation.rs @@ -1,20 +1,20 @@ use crate::Error; use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult}; -pub trait RealtimeProviderConfig { +pub trait RealtimeProviderConfig: Sync { /// Build the upstream WebSocket URL (e.g. `wss://api.openai.com/v1/realtime?model=…`). /// Pure string construction only — no network, no env. fn complete_url(&self, api_base: Option<&str>, model: &str) -> String; /// Transform a client → backend event before it is forwarded upstream. - fn transform_realtime_request( + fn transform_request( &self, event: &RealtimeEvent, model: &str, ) -> Result; /// Transform a backend → client event before it is forwarded downstream. - fn transform_realtime_response( + fn transform_response( &self, event: &RealtimeEvent, model: &str, diff --git a/litellm-rust/crates/core/src/responses/websocket.rs b/litellm-rust/crates/core/src/responses/websocket.rs index 5d037e9cf1b..28938b314f9 100644 --- a/litellm-rust/crates/core/src/responses/websocket.rs +++ b/litellm-rust/crates/core/src/responses/websocket.rs @@ -3,36 +3,28 @@ use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH} use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult}; pub trait ResponsesWebSocketProviderConfig: Sync { - fn supports_native_websocket(&self) -> bool { - false - } - fn model_in_websocket_url(&self) -> bool { true } - fn complete_websocket_url(&self, api_base: Option<&str>, model: &str) -> String { - complete_websocket_url(api_base, model, self.model_in_websocket_url()) + fn complete_url(&self, api_base: Option<&str>, model: &str) -> String { + complete_url(api_base, model, self.model_in_websocket_url()) } - fn transform_ws_request( + fn transform_request( &self, event: &ResponsesWsEvent, model: &str, ) -> Result; - fn transform_ws_response( + fn transform_response( &self, event: &ResponsesWsEvent, model: &str, ) -> Result; } -pub fn complete_websocket_url( - api_base: Option<&str>, - model: &str, - model_in_websocket_url: bool, -) -> String { +pub fn complete_url(api_base: Option<&str>, model: &str, model_in_websocket_url: bool) -> String { let base = api_base .map(str::trim) .filter(|value| !value.is_empty()) @@ -67,23 +59,10 @@ pub fn complete_websocket_url( format!( "{url}{}model={}", if query.is_some() { "&" } else { "?" }, - percent_encode(model) + crate::http_utils::encode_query_value(model) ) } -fn percent_encode(value: &str) -> String { - value - .bytes() - .map(|byte| { - if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { - format!("{}", byte as char) - } else { - format!("%{byte:02X}") - } - }) - .collect() -} - pub fn enforce_model(event: &ResponsesWsEvent, model: &str) -> ResponsesWsEvent { if !event.is_response_create() { return event.clone(); @@ -136,19 +115,19 @@ mod tests { #[test] fn url_construction_matches_python_defaults_and_query_behavior() { assert_eq!( - complete_websocket_url(None, "gpt-5", true), + complete_url(None, "gpt-5", true), "wss://api.openai.com/v1/responses?model=gpt-5" ); assert_eq!( - complete_websocket_url(Some("http://localhost:8080/"), "gpt 5", true), + complete_url(Some("http://localhost:8080/"), "gpt 5", true), "ws://localhost:8080/responses?model=gpt%205" ); assert_eq!( - complete_websocket_url(Some("https://example.test/v1?foo=bar"), "gpt-5", true), + complete_url(Some("https://example.test/v1?foo=bar"), "gpt-5", true), "wss://example.test/v1/responses?foo=bar&model=gpt-5" ); assert_eq!( - complete_websocket_url(Some("https://example.test?model=existing"), "gpt-5", true), + complete_url(Some("https://example.test?model=existing"), "gpt-5", true), "wss://example.test/responses?model=existing" ); } diff --git a/litellm-rust/crates/core/tests/anthropic_chat_completions.rs b/litellm-rust/crates/core/tests/anthropic_chat_completions.rs index aad2f466300..4ec6e1ffc83 100644 --- a/litellm-rust/crates/core/tests/anthropic_chat_completions.rs +++ b/litellm-rust/crates/core/tests/anthropic_chat_completions.rs @@ -13,19 +13,29 @@ fn params(value: Value) -> Map { } fn transform(model: &str, msgs: Value, opts: Value) -> Value { - ANTHROPIC_CHAT_COMPLETIONS_CONFIG - .transform_request(model, messages(msgs), params(opts)) - .expect("request transforms") - .body + serde_json::to_value( + ANTHROPIC_CHAT_COMPLETIONS_CONFIG + .transform_request( + model, + build_conversation(&messages(msgs)), + serde_json::from_value(opts).unwrap(), + ) + .expect("request transforms"), + ) + .unwrap() } fn transform_response(body: Value) -> Result { - ANTHROPIC_CHAT_COMPLETIONS_CONFIG - .transform_response("claude-sonnet-4-5", ProviderChatResponseData { body }) + crate::chat_completions::handler::decode_response( + &ANTHROPIC_CHAT_COMPLETIONS_CONFIG, + "claude-sonnet-4-5", + body, + "messages", + ) } fn reason(msgs: Value, opts: Value) -> Option { - ANTHROPIC_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) + ANTHROPIC_CHAT_COMPLETIONS_CONFIG.decline_reason(&messages(msgs), ¶ms(opts)) } #[test] @@ -426,7 +436,7 @@ fn resolves_the_messages_url_and_x_api_key_auth() { config .auth(Some("sk-x"), "claude-sonnet-4-5", &Map::new(), &|_| None) .expect("auth resolves"), - ChatCompletionsAuth::Header { + RequestAuth::Header { name: "x-api-key", value: "sk-x".to_string() } @@ -439,3 +449,29 @@ fn resolves_the_messages_url_and_x_api_key_auth() { ] ); } + +#[test] +fn preserves_explicit_null_mapped_params() { + let body = transform( + "model", + json!([{"role":"user","content":"hi"}]), + json!({"max_tokens":null,"temperature":null}), + ); + assert!(body.get("max_tokens").is_some_and(Value::is_null)); + assert!(body.get("temperature").is_some_and(Value::is_null)); + assert!(body.get("top_p").is_none()); +} + +#[test] +fn malformed_response_fields_do_not_expose_provider_content() { + let error = transform_response(json!({ + "model": "model", + "content": [{"type":"text","text":"private document"}], + "usage": {"input_tokens":"private token value"} + })) + .unwrap_err(); + assert_eq!( + error, + ChatResponseError::InvalidJson("invalid provider response fields".into()) + ); +} diff --git a/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs b/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs index a635ba18a59..071d1cf95cc 100644 --- a/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs +++ b/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs @@ -67,9 +67,8 @@ fn invalid_page_types_return_typed_errors(#[case] pages: Value) { fn invalid_page_values_never_panic(#[case] pages: Value) { let input = serde_json::from_value(json!({"pages":pages})).unwrap(); assert!( - AZURE_DOCUMENT_INTELLIGENCE - .format() - .map_ocr_params(input) + crate::ocr::registry::AzureDocumentIntelligence::FORMAT + .map_params(input) .is_err() ); } @@ -105,9 +104,8 @@ async fn document_intelligence_maps_features(#[case] features: Value, #[case] ex #[case(json!(""))] fn document_intelligence_rejects_invalid_features(#[case] features: Value) { assert!( - AZURE_DOCUMENT_INTELLIGENCE - .format() - .map_ocr_params(serde_json::from_value(json!({"features":features})).unwrap()) + crate::ocr::registry::AzureDocumentIntelligence::FORMAT + .map_params(serde_json::from_value(json!({"features":features})).unwrap()) .is_err() ); } diff --git a/litellm-rust/crates/core/tests/bedrock_chat_completions.rs b/litellm-rust/crates/core/tests/bedrock_chat_completions.rs index eacdba61c85..92a441df694 100644 --- a/litellm-rust/crates/core/tests/bedrock_chat_completions.rs +++ b/litellm-rust/crates/core/tests/bedrock_chat_completions.rs @@ -13,25 +13,29 @@ fn params(value: Value) -> Map { } fn transform(msgs: Value, opts: Value) -> Value { - BEDROCK_CHAT_COMPLETIONS_CONFIG - .transform_request( - "anthropic.claude-sonnet-4-5-v1:0", - messages(msgs), - params(opts), - ) - .expect("request transforms") - .body + serde_json::to_value( + BEDROCK_CHAT_COMPLETIONS_CONFIG + .transform_request( + "anthropic.claude-sonnet-4-5-v1:0", + build_conversation(&messages(msgs)), + serde_json::from_value(opts).unwrap(), + ) + .expect("request transforms"), + ) + .unwrap() } fn transform_response(body: Value) -> Result { - BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response( + crate::chat_completions::handler::decode_response( + &BEDROCK_CHAT_COMPLETIONS_CONFIG, "anthropic.claude-sonnet-4-5-v1:0", - ProviderChatResponseData { body }, + body, + "converse", ) } fn reason(msgs: Value, opts: Value) -> Option { - BEDROCK_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) + BEDROCK_CHAT_COMPLETIONS_CONFIG.decline_reason(&messages(msgs), ¶ms(opts)) } #[test] @@ -279,7 +283,7 @@ fn signs_with_sigv4_in_the_resolved_region() { &|_| None ) .expect("auth resolves"), - ChatCompletionsAuth::AwsSigV4 { + RequestAuth::AwsSigV4 { region: "eu-central-1".to_string() } ); @@ -304,10 +308,10 @@ fn a_bearer_token_outranks_sigv4_the_way_python_resolves_it() { ) .expect("auth resolves") }; - let bearer = |token: &str| ChatCompletionsAuth::Bearer { + let bearer = |token: &str| RequestAuth::Bearer { token: token.to_string(), }; - let sigv4 = ChatCompletionsAuth::AwsSigV4 { + let sigv4 = RequestAuth::AwsSigV4 { region: "eu-central-1".to_string(), }; @@ -575,3 +579,40 @@ fn host_supplied_credentials_outrank_ambient_profile_and_role_state() { ); assert!(host_supplied_credentials(&Map::new()).is_none()); } + +#[test] +fn preserves_explicit_null_mapped_params() { + let body = transform( + json!([{"role":"user","content":"hi"}]), + json!({"maxTokens":null,"temperature":null}), + ); + assert!( + body["inferenceConfig"] + .get("maxTokens") + .is_some_and(Value::is_null) + ); + assert!( + body["inferenceConfig"] + .get("temperature") + .is_some_and(Value::is_null) + ); + assert!(body["inferenceConfig"].get("topP").is_none()); +} + +#[test] +fn rejects_empty_or_mixed_converse_blocks() { + for block in [json!({}), json!({"text":"visible","toolUse":{}})] { + assert_eq!( + transform_response(json!({ + "output":{"message":{"content":[block]}},"usage":{} + })) + .unwrap_err(), + ChatResponseError::NonTextContent + ); + } + let response = transform_response(json!({ + "output":{"message":{"content":[{"text":null}]}},"usage":{} + })) + .unwrap(); + assert_eq!(response.choices[0].message.content.as_deref(), Some("")); +} diff --git a/litellm-rust/crates/core/tests/chat_completions.rs b/litellm-rust/crates/core/tests/chat_completions.rs index 15b0ee0153e..5e2e74f7bbf 100644 --- a/litellm-rust/crates/core/tests/chat_completions.rs +++ b/litellm-rust/crates/core/tests/chat_completions.rs @@ -1,9 +1,10 @@ +use crate::auth::RequestAuth; use serde_json::{Map, Value, json}; use crate::error::Error; use super::prepare::{prepare_provider_request, resolve_request}; -use super::transformation::ChatCompletionsAuth; + use super::types::{ChatCompletionsRequest, ProviderChatCompletionsRequest}; fn prepare_chat_completions_call( @@ -89,7 +90,7 @@ fn adds_the_auth_and_default_headers() { ); assert!(matches!( prepared.auth, - ChatCompletionsAuth::Header { + RequestAuth::Header { name: "x-api-key", .. } @@ -287,7 +288,7 @@ fn prepares_a_bedrock_call_without_resolving_credentials() { ); assert_eq!( prepared.auth, - ChatCompletionsAuth::AwsSigV4 { + RequestAuth::AwsSigV4 { region: "us-east-1".to_string() } ); @@ -468,7 +469,7 @@ fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() { .expect("prepares"); assert_eq!( prepared.auth, - ChatCompletionsAuth::Bearer { + RequestAuth::Bearer { token: "sk-test".to_string() } ); @@ -814,3 +815,36 @@ mod round_trip { ); } } + +#[test] +fn malformed_provider_params_decline_before_credentials_are_resolved() { + for optional_params in [json!({"max_tokens":"many"}), json!({"stop_sequences":[42]})] { + let request = ChatCompletionsRequest { + model: "anthropic/model", + custom_llm_provider: None, + messages: json!([{"role":"user","content":"hi"}]), + optional_params: optional_params.as_object().unwrap().clone(), + api_key: None, + api_base: None, + extra_headers: None, + timeout: None, + }; + assert_eq!( + super::chat_completions_decline_reason( + request.model, + None, + request.messages.clone(), + &request.optional_params + ), + Some("invalid provider-mapped parameters"), + ); + let error = match resolve_request(request) { + Ok(_) => panic!("malformed params must decline"), + Err(error) => error, + }; + assert_eq!( + error, + Error::Unsupported("invalid provider-mapped parameters") + ); + } +} diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index 91eb60d843f..b0e7920bfcb 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -25,13 +25,12 @@ pub(crate) async fn perform_ocr( ocr_client().perform(request).await } -pub(crate) fn params(integration: &I, value: Value) -> MappedParams +pub(crate) fn params(_integration: &I, value: Value) -> MappedParams where I: OcrIntegration, { - integration - .format() - .map_ocr_params(serde_json::from_value(value).unwrap()) + I::FORMAT + .map_params(serde_json::from_value(value).unwrap()) .unwrap() } pub(crate) fn transform( @@ -43,16 +42,12 @@ pub(crate) fn transform( where I: OcrIntegration, { - let params = integration - .format() - .map_ocr_params(serde_json::from_value(options).unwrap())?; + let params = I::FORMAT.map_params(serde_json::from_value(options).unwrap())?; let decoded = decode_response( &serde_json::to_vec(&value).unwrap(), integration.preserve_native_response(¶ms), )?; - let mut response = integration - .format() - .transform_ocr_response(model, decoded.data, ¶ms)?; + let mut response = I::FORMAT.transform_response(model, decoded.data, ¶ms)?; response.provider_native_response = decoded.native; Ok(response.into_json()) } @@ -65,9 +60,7 @@ pub(crate) async fn body( where I: OcrIntegration, { - let params = integration - .format() - .map_ocr_params(serde_json::from_value(options).unwrap())?; + let params = I::FORMAT.map_params(serde_json::from_value(options).unwrap())?; let client = ocr_client(); let document = integration .prepare_document( @@ -78,12 +71,8 @@ where ) .await?; Ok( - serde_json::to_value(integration.format().transform_ocr_request( - model, - document.into(), - ¶ms, - )?) - .unwrap(), + serde_json::to_value(I::FORMAT.transform_request(model, document.into(), ¶ms)?) + .unwrap(), ) } pub(crate) fn wire_request(model: &str, base: &str, options: Value) -> OcrRequest { diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/chat_completions/case.py b/tests/rust-python-harness/strategies/trace_parity/sdk/chat_completions/case.py index 6be5afd60d6..66ff8eb78b4 100644 --- a/tests/rust-python-harness/strategies/trace_parity/sdk/chat_completions/case.py +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/chat_completions/case.py @@ -18,7 +18,7 @@ COMMON_MAPPINGS: Final = ( span="python_provider_supported_openai_params", python_frame=r"AnthropicConfig\.get_supported_openai_params$", ), - mapping(rust_span="supported_openai_params"), + mapping(rust_span="supported_provider_params"), mapping(rust_span="validate_environment", python_frame=r"(? Route VERTEX_COMMON_MAPPINGS: Final = ( *COMMON_MAPPINGS[:7], mapping( - rust_span="transform_ocr_request", + rust_span="transform_request", python_frame=( r"VertexAIOCRConfig\.(?:async_)?transform_ocr_request$" r"|MistralOCRConfig\.transform_ocr_request$" @@ -264,13 +264,13 @@ VERTEX_SYNC_MAPPINGS: Final = ( *VERTEX_COMMON_MAPPINGS, mapping(rust_span="execute_ocr_provider_call", python_frame=r"BaseLLMHTTPHandler\.ocr$"), mapping(span="python_transform_ocr_response_wrapper", python_frame=r"BaseLLMHTTPHandler\._transform_ocr_response$"), - mapping(rust_span="transform_ocr_response", python_frame=r"MistralOCRConfig\.transform_ocr_response$"), + mapping(rust_span="transform_response", python_frame=r"MistralOCRConfig\.transform_ocr_response$"), ) VERTEX_ASYNC_MAPPINGS: Final = ( *VERTEX_COMMON_MAPPINGS, mapping(span="python_ocr_wrapper", python_frame=r"BaseLLMHTTPHandler\.ocr$"), mapping(rust_span="execute_ocr_provider_call", python_frame=r"BaseLLMHTTPHandler\.async_ocr$"), - mapping(rust_span="transform_ocr_response", python_frame=r"MistralOCRConfig\.transform_ocr_response$"), + mapping(rust_span="transform_response", python_frame=r"MistralOCRConfig\.transform_ocr_response$"), ) DEEPSEEK_COMMON_MAPPINGS: Final = ( @@ -278,16 +278,16 @@ DEEPSEEK_COMMON_MAPPINGS: Final = ( mapping(rust_span="prepare_ocr_call", python_frame=r"ocr/main\.py:\d+ _prepare_ocr_request$"), mapping(rust_span="ocr_provider_config", python_frame=r"ProviderConfigManager\.get_provider_ocr_config$"), mapping(rust_span="supported_ocr_params", python_frame=r"get_supported_ocr_params$"), - mapping(rust_span="map_ocr_params", python_frame=r"(?