mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
refactor(core): tighten provider trait contracts
This commit is contained in:
parent
7107f2ebbd
commit
601f45f9a9
56 changed files with 992 additions and 746 deletions
|
|
@ -153,7 +153,7 @@ where
|
|||
// Relay a buffered backend event (warm handoff's session.created) first, so a
|
||||
// warm session looks identical to a fresh one from the client's view.
|
||||
if let Some(event) = prelude {
|
||||
for outbound in config.transform_realtime_response(&event, model)?.events {
|
||||
for outbound in config.transform_response(&event, model)?.events {
|
||||
client_out
|
||||
.send(outbound)
|
||||
.await
|
||||
|
|
@ -176,7 +176,7 @@ where
|
|||
// (carrying usage) are server→client events; observing the client arm
|
||||
// would let an authenticated client POST a fabricated response.done and
|
||||
// inflate its own spend log. Logging observes upstream events only.
|
||||
for outbound in config.transform_realtime_request(&event, model)?.events {
|
||||
for outbound in config.transform_request(&event, model)?.events {
|
||||
let payload = serde_json::to_string(&outbound)
|
||||
.map_err(|err| Error::InvalidResponse(err.to_string()))?;
|
||||
upstream_tx
|
||||
|
|
@ -193,7 +193,7 @@ where
|
|||
let event: RealtimeEvent = serde_json::from_str(&text)
|
||||
.map_err(|err| Error::InvalidResponse(err.to_string()))?;
|
||||
observe(&event);
|
||||
for outbound in config.transform_realtime_response(&event, model)?.events {
|
||||
for outbound in config.transform_response(&event, model)?.events {
|
||||
client_out
|
||||
.send(outbound)
|
||||
.await
|
||||
|
|
@ -212,8 +212,8 @@ where
|
|||
}
|
||||
|
||||
/// Splice a client realtime stream to OpenAI: forward client events upstream
|
||||
/// (via `transform_realtime_request`) and backend events downstream (via
|
||||
/// `transform_realtime_response`). Returns when either side closes.
|
||||
/// (via `transform_request`) and backend events downstream (via
|
||||
/// `transform_response`). Returns when either side closes.
|
||||
///
|
||||
/// Generic over the client transport (typed events) so this crate stays
|
||||
/// framework-agnostic; the gateway adapts its axum socket to these. This is the
|
||||
|
|
|
|||
|
|
@ -130,7 +130,7 @@ async fn dial_upstream(
|
|||
api_key: &str,
|
||||
api_base: Option<&str>,
|
||||
) -> Result<ResponsesUpstreamWs, Error> {
|
||||
let url = OPENAI_RESPONSES_WS_CONFIG.complete_websocket_url(api_base, model);
|
||||
let url = OPENAI_RESPONSES_WS_CONFIG.complete_url(api_base, model);
|
||||
let mut request = url
|
||||
.as_str()
|
||||
.into_client_request()
|
||||
|
|
@ -208,7 +208,7 @@ where
|
|||
event = client_in.next() => {
|
||||
let Some(event) = event else { break };
|
||||
for outbound in OPENAI_RESPONSES_WS_CONFIG
|
||||
.transform_ws_request(&event, model)?
|
||||
.transform_request(&event, model)?
|
||||
.events
|
||||
{
|
||||
let payload = serde_json::to_string(&outbound)
|
||||
|
|
@ -226,7 +226,7 @@ where
|
|||
.map_err(|error| Error::InvalidResponse(error.to_string()))?;
|
||||
observe(&event);
|
||||
for outbound in OPENAI_RESPONSES_WS_CONFIG
|
||||
.transform_ws_response(&event, model)?
|
||||
.transform_response(&event, model)?
|
||||
.events
|
||||
{
|
||||
client_out.send(outbound)
|
||||
|
|
@ -265,7 +265,7 @@ where
|
|||
let (mut upstream_tx, upstream_rx) = upstream.split();
|
||||
if let Some(first_frame) = first_frame {
|
||||
for outbound in OPENAI_RESPONSES_WS_CONFIG
|
||||
.transform_ws_request(&first_frame, model)?
|
||||
.transform_request(&first_frame, model)?
|
||||
.events
|
||||
{
|
||||
let payload = serde_json::to_string(&outbound)
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
use crate::auth::RequestAuth;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::error::Error;
|
||||
|
|
@ -38,7 +39,7 @@ pub async fn execute_audio_transcription_provider_call(
|
|||
.map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?;
|
||||
Ok(request
|
||||
.config
|
||||
.transform_transcription_response(&request.model, response_json)?
|
||||
.transform_response(&request.model, response_json)?
|
||||
.into_json())
|
||||
}
|
||||
|
||||
|
|
@ -50,11 +51,10 @@ async fn signed_headers(
|
|||
use std::collections::BTreeMap;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use crate::audio_transcription::transformation::AudioTranscriptionAuth;
|
||||
use crate::providers::bedrock::audio_transcription::aws_auth_config;
|
||||
use crate::providers::bedrock::aws_base::{resolve_credentials, sign_bedrock_post};
|
||||
|
||||
let AudioTranscriptionAuth::AwsSigV4 { region, .. } = &request.auth else {
|
||||
let RequestAuth::AwsSigV4 { region, .. } = &request.auth else {
|
||||
return Ok(request.upstream_headers.clone());
|
||||
};
|
||||
let env_lookup = |key: &str| std::env::var(key).ok();
|
||||
|
|
@ -80,12 +80,12 @@ async fn signed_headers(
|
|||
request: &ProviderAudioTranscriptionRequest,
|
||||
_body: &[u8],
|
||||
) -> Result<Vec<(String, String)>, Error> {
|
||||
use crate::audio_transcription::transformation::AudioTranscriptionAuth;
|
||||
|
||||
match request.auth {
|
||||
AudioTranscriptionAuth::AwsSigV4 { .. } => Err(Error::Unsupported(
|
||||
RequestAuth::AwsSigV4 { .. } => Err(Error::Unsupported(
|
||||
"AWS SigV4 requires the bedrock-auth feature",
|
||||
)),
|
||||
AudioTranscriptionAuth::Bearer => Ok(request.upstream_headers.clone()),
|
||||
RequestAuth::Bearer { .. } | RequestAuth::Header { .. } => {
|
||||
Ok(request.upstream_headers.clone())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
use crate::auth::RequestAuth;
|
||||
use crate::error::Error;
|
||||
use crate::http_utils::{has_header, string_headers};
|
||||
#[cfg(feature = "bedrock-auth")]
|
||||
use crate::providers::bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG;
|
||||
use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
|
||||
|
||||
use super::transformation::{AudioTranscriptionAuth, AudioTranscriptionProviderConfig};
|
||||
use super::transformation::AudioTranscriptionProviderConfig;
|
||||
use super::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest};
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
|
|
@ -40,12 +41,20 @@ pub fn prepare_audio_transcription_provider_call(
|
|||
.ok_or_else(|| Error::InvalidProvider(provider_info.custom_llm_provider.to_string()))?;
|
||||
let env_lookup = |key: &str| std::env::var(key).ok();
|
||||
let mut headers = string_headers("audio transcription", request.extra_headers)?;
|
||||
let auth = config.auth_strategy(&model, &request.optional_params, &env_lookup)?;
|
||||
if matches!(auth, AudioTranscriptionAuth::Bearer)
|
||||
&& !has_header(&headers, "authorization")
|
||||
&& let Some(api_key) = request.api_key
|
||||
{
|
||||
headers.push(("Authorization".to_string(), format!("Bearer {api_key}")));
|
||||
let auth = config.auth(
|
||||
request.api_key,
|
||||
&model,
|
||||
&request.optional_params,
|
||||
&env_lookup,
|
||||
)?;
|
||||
match &auth {
|
||||
RequestAuth::Bearer { token } if !has_header(&headers, "authorization") => {
|
||||
headers.push(("Authorization".into(), format!("Bearer {token}")));
|
||||
}
|
||||
RequestAuth::Header { name, value } if !has_header(&headers, name) => {
|
||||
headers.push(((*name).into(), value.clone()));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
if !has_header(&headers, "content-type") {
|
||||
headers.push(("Content-Type".to_string(), "application/json".to_string()));
|
||||
|
|
@ -56,15 +65,18 @@ pub fn prepare_audio_transcription_provider_call(
|
|||
&request.optional_params,
|
||||
&env_lookup,
|
||||
)?;
|
||||
let filtered_params = config.map_transcription_params(&request.optional_params);
|
||||
let transformed =
|
||||
config.transform_transcription_request(&model, request.audio, filtered_params)?;
|
||||
let audio = serde_json::from_value(request.audio)
|
||||
.map_err(|_| Error::InvalidRequest("invalid audio data or format".into()))?;
|
||||
let params = map_params(request.optional_params.clone())?;
|
||||
let transformed = config.transform_request(&model, audio, params)?;
|
||||
let body = serde_json::to_value(transformed)
|
||||
.map_err(|_| Error::InvalidRequest("invalid audio request body".into()))?;
|
||||
Ok(ProviderAudioTranscriptionRequest {
|
||||
model,
|
||||
custom_llm_provider: provider_info.custom_llm_provider.to_string(),
|
||||
config,
|
||||
url,
|
||||
body: transformed.body,
|
||||
body,
|
||||
upstream_headers: headers,
|
||||
auth,
|
||||
#[cfg(feature = "bedrock-auth")]
|
||||
|
|
@ -72,3 +84,11 @@ pub fn prepare_audio_transcription_provider_call(
|
|||
timeout: request.timeout,
|
||||
})
|
||||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn map_params(
|
||||
params: serde_json::Map<String, serde_json::Value>,
|
||||
) -> Result<super::types::TranscriptionParams, Error> {
|
||||
serde_json::from_value(serde_json::Value::Object(params))
|
||||
.map_err(|_| Error::InvalidRequest("invalid transcription parameters".into()))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,44 +1,20 @@
|
|||
use crate::Error;
|
||||
use crate::auth::RequestAuth;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::types::{AudioTranscriptionRequestData, AudioTranscriptionResponseData};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum AudioTranscriptionAuth {
|
||||
Bearer,
|
||||
AwsSigV4 {
|
||||
region: String,
|
||||
service: &'static str,
|
||||
},
|
||||
}
|
||||
use super::types::{
|
||||
AudioTranscriptionRequestData, AudioTranscriptionResponseData, ProviderTranscriptionResponse,
|
||||
TranscriptionAudio, TranscriptionParams,
|
||||
};
|
||||
|
||||
pub trait AudioTranscriptionProviderConfig: Sync {
|
||||
fn supported_transcription_params(&self) -> &'static [&'static str];
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn map_transcription_params(&self, params: &Map<String, Value>) -> Map<String, Value> {
|
||||
params
|
||||
.iter()
|
||||
.filter(|(key, _)| {
|
||||
self.supported_transcription_params()
|
||||
.contains(&key.as_str())
|
||||
})
|
||||
.map(|(key, value)| (key.clone(), value.clone()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn transform_transcription_request(
|
||||
fn auth(
|
||||
&self,
|
||||
api_key: Option<&str>,
|
||||
model: &str,
|
||||
audio: Value,
|
||||
optional_params: Map<String, Value>,
|
||||
) -> Result<AudioTranscriptionRequestData, Error>;
|
||||
|
||||
fn transform_transcription_response(
|
||||
&self,
|
||||
model: &str,
|
||||
response_json: Value,
|
||||
) -> Result<AudioTranscriptionResponseData, Error>;
|
||||
optional_params: &Map<String, Value>,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<RequestAuth, Error>;
|
||||
|
||||
fn complete_url(
|
||||
&self,
|
||||
|
|
@ -48,10 +24,16 @@ pub trait AudioTranscriptionProviderConfig: Sync {
|
|||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error>;
|
||||
|
||||
fn auth_strategy(
|
||||
fn transform_request(
|
||||
&self,
|
||||
model: &str,
|
||||
optional_params: &Map<String, Value>,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<AudioTranscriptionAuth, Error>;
|
||||
audio: TranscriptionAudio,
|
||||
params: TranscriptionParams,
|
||||
) -> Result<AudioTranscriptionRequestData, Error>;
|
||||
|
||||
fn transform_response(
|
||||
&self,
|
||||
model: &str,
|
||||
response: ProviderTranscriptionResponse,
|
||||
) -> Result<AudioTranscriptionResponseData, Error>;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
use crate::auth::RequestAuth;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::transformation::{AudioTranscriptionAuth, AudioTranscriptionProviderConfig};
|
||||
use super::transformation::AudioTranscriptionProviderConfig;
|
||||
|
||||
pub struct AudioTranscriptionRequest<'a> {
|
||||
pub model: &'a str,
|
||||
|
|
@ -24,7 +25,7 @@ pub struct ProviderAudioTranscriptionRequest {
|
|||
pub(super) url: String,
|
||||
pub(super) body: Value,
|
||||
pub(super) upstream_headers: Vec<(String, String)>,
|
||||
pub(super) auth: AudioTranscriptionAuth,
|
||||
pub(super) auth: RequestAuth,
|
||||
#[cfg(feature = "bedrock-auth")]
|
||||
pub(super) optional_params: Map<String, Value>,
|
||||
pub(super) timeout: Option<Duration>,
|
||||
|
|
@ -52,9 +53,43 @@ impl ProviderAudioTranscriptionRequest {
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct AudioTranscriptionRequestData {
|
||||
pub body: Value,
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum AudioFormat {
|
||||
Wav,
|
||||
Mp3,
|
||||
Flac,
|
||||
Ogg,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct TranscriptionAudio {
|
||||
pub data: String,
|
||||
pub format: AudioFormat,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
pub struct TranscriptionParams {
|
||||
pub language: Option<String>,
|
||||
pub prompt: Option<String>,
|
||||
#[serde(
|
||||
default,
|
||||
deserialize_with = "crate::http_utils::deserialize_optional_param"
|
||||
)]
|
||||
pub temperature: Option<Option<serde_json::Number>>,
|
||||
pub response_format: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum AudioTranscriptionRequestData {
|
||||
Bedrock(crate::providers::bedrock::converse::ConverseRequest),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ProviderTranscriptionResponse {
|
||||
Bedrock(crate::providers::bedrock::converse::ConverseResponse),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use crate::AuthError;
|
|||
use crate::auth::error::AuthConfigurationError;
|
||||
use crate::auth::{
|
||||
CredentialFileRef, CredentialLookup, CredentialRef, ResolvedCredential, SecretValue,
|
||||
TokenCallerHandle,
|
||||
TokenProviderHandle,
|
||||
};
|
||||
|
||||
use super::native::{NativeAzureRequest, NativeAzureTokenAcquirer};
|
||||
|
|
@ -20,7 +20,7 @@ const AZURE_FEDERATED_TOKEN_FILE_ENV: &str = "AZURE_FEDERATED_TOKEN_FILE";
|
|||
#[derive(Clone, Debug)]
|
||||
pub(crate) enum AzureCredentialPlan {
|
||||
Supplied(ResolvedCredential),
|
||||
Caller(TokenCallerHandle),
|
||||
Caller(TokenProviderHandle),
|
||||
Oidc {
|
||||
reference: CredentialRef,
|
||||
tenant_id: String,
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ use serde_json::{Map, Value};
|
|||
use strum::EnumString;
|
||||
|
||||
use crate::AuthError;
|
||||
use crate::auth::{CredentialResolverHandle, SecretValue, TokenCallerHandle};
|
||||
use crate::auth::{CredentialResolverHandle, SecretValue, TokenProviderHandle};
|
||||
|
||||
pub const DEFAULT_AZURE_SCOPE: &str = "https://cognitiveservices.azure.com/.default";
|
||||
|
||||
|
|
@ -36,7 +36,7 @@ pub enum AzureCredentialType {
|
|||
#[derive(Clone, Debug, Default)]
|
||||
pub struct AzureAuthInputs {
|
||||
pub azure_ad_token: ConfigValue<SecretValue>,
|
||||
pub azure_ad_token_provider: Option<TokenCallerHandle>,
|
||||
pub azure_ad_token_provider: Option<TokenProviderHandle>,
|
||||
pub credential_resolver: Option<CredentialResolverHandle>,
|
||||
pub tenant_id: ConfigValue<String>,
|
||||
pub client_id: ConfigValue<String>,
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ use veil::Redact;
|
|||
|
||||
use crate::error::AuthError;
|
||||
|
||||
use super::{ResolvedCredential, SecretValue, TokenCallerHandle};
|
||||
use super::{ResolvedCredential, SecretValue, TokenProviderHandle};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum CredentialFileRef {
|
||||
|
|
@ -55,7 +55,7 @@ impl CredentialResolverHandle {
|
|||
#[derive(Clone, Debug)]
|
||||
pub enum CredentialPlan {
|
||||
Static(CredentialRef),
|
||||
Caller(TokenCallerHandle),
|
||||
Caller(TokenProviderHandle),
|
||||
None,
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,15 @@ pub(crate) fn apply_credential(
|
|||
)
|
||||
}
|
||||
|
||||
/// How the upstream call is authenticated. API-key strategies are resolved in
|
||||
/// `prepare`; SigV4 needs the serialized body, so the handler signs it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum RequestAuth {
|
||||
Header { name: &'static str, value: String },
|
||||
Bearer { token: String },
|
||||
AwsSigV4 { region: String },
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{CredentialPlacement, apply_credential};
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ pub use credential::{
|
|||
CredentialFileRef, CredentialLookup, CredentialLookupFuture, CredentialPlan,
|
||||
CredentialPlanResolution, CredentialRef, CredentialResolver, CredentialResolverHandle,
|
||||
};
|
||||
pub use http::CredentialPlacement;
|
||||
pub use http::{CredentialPlacement, RequestAuth};
|
||||
pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
|
||||
pub use secret::SecretValue;
|
||||
pub use token::{ResolvedCredential, TokenCaller, TokenCallerHandle, TokenFuture};
|
||||
pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};
|
||||
|
|
|
|||
|
|
@ -29,15 +29,15 @@ impl ResolvedCredential {
|
|||
pub type TokenFuture<'a> =
|
||||
Pin<Box<dyn Future<Output = Result<ResolvedCredential, AuthError>> + Send + 'a>>;
|
||||
|
||||
pub trait TokenCaller: std::fmt::Debug + Send + Sync {
|
||||
pub trait TokenProvider: std::fmt::Debug + Send + Sync {
|
||||
fn acquire(&self) -> TokenFuture<'_>;
|
||||
}
|
||||
|
||||
#[derive(Clone, Redact)]
|
||||
pub struct TokenCallerHandle(#[redact(with = "[REDACTED]")] Arc<dyn TokenCaller>);
|
||||
pub struct TokenProviderHandle(#[redact(with = "[REDACTED]")] Arc<dyn TokenProvider>);
|
||||
|
||||
impl TokenCallerHandle {
|
||||
pub fn new(caller: Arc<dyn TokenCaller>) -> Self {
|
||||
impl TokenProviderHandle {
|
||||
pub fn new(caller: Arc<dyn TokenProvider>) -> Self {
|
||||
Self(caller)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -7,20 +7,52 @@ use super::transformation::ChatCompletionsProviderConfig;
|
|||
|
||||
const HEADER_CONTEXT: &str = "chat completions";
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
pub(super) fn chat_completions_provider_config(
|
||||
provider: &str,
|
||||
) -> Option<&'static dyn ChatCompletionsProviderConfig> {
|
||||
#[derive(Clone, Copy)]
|
||||
pub(super) enum ChatProviderConfig {
|
||||
Anthropic,
|
||||
#[cfg(feature = "bedrock-auth")]
|
||||
Bedrock,
|
||||
}
|
||||
|
||||
pub(super) fn chat_completions_provider_config(provider: &str) -> Option<ChatProviderConfig> {
|
||||
match provider {
|
||||
"anthropic" => Some(&ANTHROPIC_CHAT_COMPLETIONS_CONFIG),
|
||||
"anthropic" => Some(ChatProviderConfig::Anthropic),
|
||||
#[cfg(feature = "bedrock-auth")]
|
||||
"bedrock" => Some(
|
||||
&crate::providers::bedrock::chat_completions::transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
|
||||
),
|
||||
"bedrock" => Some(ChatProviderConfig::Bedrock),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
impl ChatProviderConfig {
|
||||
pub(super) fn decline_reason(
|
||||
self,
|
||||
messages: &[super::types::ChatMessage],
|
||||
params: &Map<String, Value>,
|
||||
) -> Option<super::transformation::Unsupported> {
|
||||
match self {
|
||||
Self::Anthropic => decline_reason(&ANTHROPIC_CHAT_COMPLETIONS_CONFIG, messages, params),
|
||||
#[cfg(feature = "bedrock-auth")]
|
||||
Self::Bedrock => decline_reason(
|
||||
&crate::providers::bedrock::chat_completions::transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
|
||||
messages,
|
||||
params,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decline_reason<C: ChatCompletionsProviderConfig>(
|
||||
config: &C,
|
||||
messages: &[super::types::ChatMessage],
|
||||
params: &Map<String, Value>,
|
||||
) -> Option<super::transformation::Unsupported> {
|
||||
config.decline_reason(messages, params).or_else(|| {
|
||||
serde_json::from_value::<C::MappedParams>(Value::Object(params.clone()))
|
||||
.err()
|
||||
.map(|_| super::transformation::Unsupported("invalid provider-mapped parameters"))
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn string_headers(
|
||||
extra_headers: Option<Map<String, Value>>,
|
||||
) -> Result<Vec<(String, String)>, Error> {
|
||||
|
|
|
|||
|
|
@ -14,7 +14,8 @@ use crate::constants::EMPTY_TEXT_PLACEHOLDER;
|
|||
|
||||
use super::types::{ChatMessage, ChatMessageContent};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum TurnRole {
|
||||
User,
|
||||
Assistant,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use super::error::{ChatRequestError, ChatResponseError};
|
||||
use crate::auth::RequestAuth;
|
||||
use crate::error::TransportError;
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
@ -7,10 +8,9 @@ use crate::http_utils::{http_request, truncate_error_body};
|
|||
|
||||
use super::client::http_client;
|
||||
use super::prepare::prepare_provider_request;
|
||||
use super::transformation::ChatCompletionsAuth;
|
||||
|
||||
use super::types::{
|
||||
ChatCompletionsResponse, ProviderChatCompletionsRequest, ProviderChatResponseData,
|
||||
ResolvedChatCompletionsRequest,
|
||||
ChatCompletionsResponse, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest,
|
||||
};
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
|
|
@ -49,10 +49,36 @@ pub(super) async fn execute_chat_completions_provider_call(
|
|||
|
||||
let body: Value = serde_json::from_str(&text)
|
||||
.map_err(|err| ChatResponseError::InvalidJson(err.to_string()))?;
|
||||
request
|
||||
.config
|
||||
.transform_response(&request.model, ProviderChatResponseData { body })
|
||||
.map_err(Error::from)
|
||||
use super::common_utils::ChatProviderConfig;
|
||||
match request.config {
|
||||
ChatProviderConfig::Anthropic => decode_response(
|
||||
&crate::providers::anthropic::chat_completions::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG,
|
||||
&request.model,
|
||||
body,
|
||||
"messages",
|
||||
),
|
||||
#[cfg(feature = "bedrock-auth")]
|
||||
ChatProviderConfig::Bedrock => decode_response(
|
||||
&crate::providers::bedrock::chat_completions::transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
|
||||
&request.model,
|
||||
body,
|
||||
"converse",
|
||||
),
|
||||
}.map_err(Error::from)
|
||||
}
|
||||
|
||||
pub(crate) fn decode_response<C: super::transformation::ChatCompletionsProviderConfig>(
|
||||
config: &C,
|
||||
model: &str,
|
||||
body: Value,
|
||||
api: &'static str,
|
||||
) -> Result<ChatCompletionsResponse, ChatResponseError> {
|
||||
if !body.is_object() {
|
||||
return Err(ChatResponseError::NotObject { api });
|
||||
}
|
||||
let response = serde_json::from_value(body)
|
||||
.map_err(|_| ChatResponseError::InvalidJson("invalid provider response fields".into()))?;
|
||||
config.transform_response(model, response)
|
||||
}
|
||||
|
||||
#[cfg(feature = "bedrock-auth")]
|
||||
|
|
@ -68,7 +94,7 @@ pub(super) async fn signed_headers(
|
|||
is_sigv4_computed_header, resolve_credentials, sign_bedrock_post,
|
||||
};
|
||||
|
||||
let ChatCompletionsAuth::AwsSigV4 { region } = &request.auth else {
|
||||
let RequestAuth::AwsSigV4 { region } = &request.auth else {
|
||||
return Ok(request.upstream_headers.clone());
|
||||
};
|
||||
// Reattaching a header the signer also emits would put both copies on the
|
||||
|
|
@ -118,7 +144,7 @@ pub(super) async fn signed_headers(
|
|||
_body: &[u8],
|
||||
) -> Result<Vec<(String, String)>, Error> {
|
||||
match &request.auth {
|
||||
ChatCompletionsAuth::AwsSigV4 { .. } => Err(Error::Unsupported(
|
||||
RequestAuth::AwsSigV4 { .. } => Err(Error::Unsupported(
|
||||
"AWS SigV4 requires the bedrock-auth feature",
|
||||
)),
|
||||
_ => Ok(request.upstream_headers.clone()),
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ pub fn chat_completions_decline_reason(
|
|||
return Some("empty message list");
|
||||
}
|
||||
config
|
||||
.unsupported_reason(&messages, optional_params)
|
||||
.decline_reason(&messages, optional_params)
|
||||
.map(|reason| reason.0)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
use super::error::ChatRequestError;
|
||||
use crate::auth::RequestAuth;
|
||||
use serde_json::Value;
|
||||
|
||||
use crate::error::Error;
|
||||
|
|
@ -6,7 +7,7 @@ use crate::http_utils::has_header;
|
|||
use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
|
||||
|
||||
use super::common_utils::{chat_completions_provider_config, string_headers};
|
||||
use super::transformation::{ChatCompletionsAuth, ChatCompletionsProviderConfig};
|
||||
use super::transformation::ChatCompletionsProviderConfig;
|
||||
use super::types::{
|
||||
ChatCompletionsRequest, ChatMessage, ProviderChatCompletionsRequest,
|
||||
ResolvedChatCompletionsRequest,
|
||||
|
|
@ -15,7 +16,7 @@ use super::types::{
|
|||
pub(super) fn resolve_provider_config<'a>(
|
||||
model: &'a str,
|
||||
custom_llm_provider: Option<&'a str>,
|
||||
) -> Result<(String, &'static dyn ChatCompletionsProviderConfig), Error> {
|
||||
) -> Result<(String, super::common_utils::ChatProviderConfig), Error> {
|
||||
let provider_info = get_custom_llm_provider(model, custom_llm_provider)
|
||||
.or_else(|| {
|
||||
custom_llm_provider.map(|provider| CustomLlmProvider {
|
||||
|
|
@ -46,7 +47,7 @@ pub(super) fn resolve_request(
|
|||
if messages.is_empty() {
|
||||
return Err(ChatRequestError::EmptyMessages.into());
|
||||
}
|
||||
if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) {
|
||||
if let Some(reason) = config.decline_reason(&messages, &request.optional_params) {
|
||||
return Err(Error::Unsupported(reason.0));
|
||||
}
|
||||
Ok(ResolvedChatCompletionsRequest {
|
||||
|
|
@ -62,11 +63,11 @@ pub(super) fn resolve_request(
|
|||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn validate_environment(
|
||||
fn validate_environment<C: ChatCompletionsProviderConfig>(
|
||||
request: &ResolvedChatCompletionsRequest<'_>,
|
||||
model: &str,
|
||||
config: &dyn ChatCompletionsProviderConfig,
|
||||
) -> Result<(Vec<(String, String)>, ChatCompletionsAuth), Error> {
|
||||
config: &C,
|
||||
) -> Result<(Vec<(String, String)>, RequestAuth), Error> {
|
||||
let env_lookup = |key: &str| std::env::var(key).ok();
|
||||
let mut headers = string_headers(request.extra_headers.clone())?;
|
||||
let auth = config.auth(
|
||||
|
|
@ -76,7 +77,7 @@ fn validate_environment(
|
|||
&env_lookup,
|
||||
)?;
|
||||
match &auth {
|
||||
ChatCompletionsAuth::Header { name, value } => {
|
||||
RequestAuth::Header { name, value } => {
|
||||
// The deployment's credential replaces whatever the caller forwarded
|
||||
// under the same name, mirroring Python's
|
||||
// `{**headers, **anthropic_headers}`: letting a request header win
|
||||
|
|
@ -91,7 +92,7 @@ fn validate_environment(
|
|||
headers.push(((*name).to_string(), value.clone()));
|
||||
}
|
||||
}
|
||||
ChatCompletionsAuth::Bearer { token } => {
|
||||
RequestAuth::Bearer { token } => {
|
||||
// Bedrock's `get_request_headers` assigns `headers["Authorization"]`
|
||||
// unconditionally once a bearer token resolves, so the deployment's
|
||||
// identity outranks whatever the caller forwarded. Keeping the
|
||||
|
|
@ -104,7 +105,7 @@ fn validate_environment(
|
|||
headers.push(("authorization".to_string(), format!("Bearer {token}")));
|
||||
}
|
||||
// SigV4 signs the serialized body, so the handler adds its headers.
|
||||
ChatCompletionsAuth::AwsSigV4 { .. } => {}
|
||||
RequestAuth::AwsSigV4 { .. } => {}
|
||||
}
|
||||
|
||||
for (name, value) in config.default_headers() {
|
||||
|
|
@ -118,24 +119,46 @@ fn validate_environment(
|
|||
pub(super) fn prepare_provider_request(
|
||||
request: ResolvedChatCompletionsRequest<'_>,
|
||||
) -> Result<ProviderChatCompletionsRequest, Error> {
|
||||
let (headers, auth) = validate_environment(&request, &request.model, request.config)?;
|
||||
let model = request.model;
|
||||
let config = request.config;
|
||||
let env_lookup = |key: &str| std::env::var(key).ok();
|
||||
use super::common_utils::ChatProviderConfig;
|
||||
match request.config {
|
||||
ChatProviderConfig::Anthropic => prepare_for(
|
||||
request,
|
||||
&crate::providers::anthropic::chat_completions::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG,
|
||||
),
|
||||
#[cfg(feature = "bedrock-auth")]
|
||||
ChatProviderConfig::Bedrock => prepare_for(
|
||||
request,
|
||||
&crate::providers::bedrock::chat_completions::transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare_for<C: ChatCompletionsProviderConfig>(
|
||||
request: ResolvedChatCompletionsRequest<'_>,
|
||||
config: &C,
|
||||
) -> Result<ProviderChatCompletionsRequest, Error> {
|
||||
let params =
|
||||
serde_json::from_value::<C::MappedParams>(Value::Object(request.optional_params.clone()))
|
||||
.map_err(|_| Error::Unsupported("invalid provider-mapped parameters"))?;
|
||||
let (headers, auth) = validate_environment(&request, &request.model, config)?;
|
||||
let url = config.complete_url(
|
||||
request.api_base,
|
||||
&model,
|
||||
&request.model,
|
||||
&request.optional_params,
|
||||
&env_lookup,
|
||||
&|key| std::env::var(key).ok(),
|
||||
)?;
|
||||
let transformed =
|
||||
config.transform_request(&model, request.messages, request.optional_params.clone())?;
|
||||
|
||||
let transformed = config.transform_request(
|
||||
&request.model,
|
||||
super::conversation::build_conversation(&request.messages),
|
||||
params,
|
||||
)?;
|
||||
let body = serde_json::to_value(transformed)
|
||||
.map_err(|error| ChatRequestError::Serialization(error.to_string()))?;
|
||||
Ok(ProviderChatCompletionsRequest {
|
||||
model,
|
||||
config,
|
||||
model: request.model,
|
||||
config: request.config,
|
||||
url,
|
||||
body: transformed.body,
|
||||
body,
|
||||
upstream_headers: headers,
|
||||
auth,
|
||||
optional_params: request.optional_params,
|
||||
|
|
|
|||
|
|
@ -1,20 +1,11 @@
|
|||
use super::conversation::Conversation;
|
||||
use crate::Error;
|
||||
use crate::auth::RequestAuth;
|
||||
use crate::chat_completions::error::{ChatRequestError, ChatResponseError};
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::types::{
|
||||
ChatCompletionsResponse, ChatMessage, ChatMessageContent, ProviderChatRequestData,
|
||||
ProviderChatResponseData,
|
||||
};
|
||||
|
||||
/// How the upstream call is authenticated. API-key strategies are resolved in
|
||||
/// `prepare`; SigV4 needs the serialized body, so the handler signs it.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub enum ChatCompletionsAuth {
|
||||
Header { name: &'static str, value: String },
|
||||
Bearer { token: String },
|
||||
AwsSigV4 { region: String },
|
||||
}
|
||||
use super::types::{ChatCompletionsResponse, ChatMessage, ChatMessageContent};
|
||||
|
||||
/// Why a request cannot be served by the Rust path.
|
||||
///
|
||||
|
|
@ -33,13 +24,31 @@ pub const STREAM_PARAM: &str = "stream";
|
|||
const IGNORABLE_MESSAGE_FIELDS: &[&str] = &["name"];
|
||||
|
||||
pub trait ChatCompletionsProviderConfig: Sync {
|
||||
fn complete_url(
|
||||
type MappedParams: DeserializeOwned;
|
||||
type RequestBody: Serialize;
|
||||
type ResponseBody: DeserializeOwned;
|
||||
|
||||
/// Accepted parameter names after Python provider mapping.
|
||||
fn supported_provider_params(&self) -> &'static [&'static str];
|
||||
|
||||
/// Parameters consumed as call configuration (credentials, endpoints)
|
||||
/// rather than placed in the body. Accepted, never serialized.
|
||||
fn config_params(&self) -> &'static [&'static str] {
|
||||
&[]
|
||||
}
|
||||
|
||||
fn decline_reason(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
model: &str,
|
||||
messages: &[ChatMessage],
|
||||
optional_params: &Map<String, Value>,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error>;
|
||||
) -> Option<Unsupported> {
|
||||
unsupported_param(
|
||||
self.supported_provider_params(),
|
||||
self.config_params(),
|
||||
optional_params,
|
||||
)
|
||||
.or_else(|| messages.iter().find_map(unsupported_message))
|
||||
}
|
||||
|
||||
fn auth(
|
||||
&self,
|
||||
|
|
@ -47,7 +56,7 @@ pub trait ChatCompletionsProviderConfig: Sync {
|
|||
model: &str,
|
||||
optional_params: &Map<String, Value>,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ChatCompletionsAuth, Error>;
|
||||
) -> Result<RequestAuth, Error>;
|
||||
|
||||
fn default_headers(&self) -> &'static [(&'static str, &'static str)] {
|
||||
&[("content-type", "application/json")]
|
||||
|
|
@ -63,44 +72,30 @@ pub trait ChatCompletionsProviderConfig: Sync {
|
|||
false
|
||||
}
|
||||
|
||||
/// Supported OpenAI parameter names paired with their provider names.
|
||||
fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)];
|
||||
|
||||
/// Parameters consumed as call configuration (credentials, endpoints)
|
||||
/// rather than placed in the body. Accepted, never serialized.
|
||||
fn config_params(&self) -> &'static [&'static str] {
|
||||
&[]
|
||||
}
|
||||
|
||||
fn unsupported_reason(
|
||||
fn complete_url(
|
||||
&self,
|
||||
messages: &[ChatMessage],
|
||||
api_base: Option<&str>,
|
||||
model: &str,
|
||||
optional_params: &Map<String, Value>,
|
||||
) -> Option<Unsupported> {
|
||||
unsupported_param(
|
||||
self.supported_openai_params(),
|
||||
self.config_params(),
|
||||
optional_params,
|
||||
)
|
||||
.or_else(|| messages.iter().find_map(unsupported_message))
|
||||
}
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error>;
|
||||
|
||||
fn transform_request(
|
||||
&self,
|
||||
model: &str,
|
||||
messages: Vec<ChatMessage>,
|
||||
optional_params: Map<String, Value>,
|
||||
) -> Result<ProviderChatRequestData, ChatRequestError>;
|
||||
conversation: Conversation,
|
||||
params: Self::MappedParams,
|
||||
) -> Result<Self::RequestBody, ChatRequestError>;
|
||||
|
||||
fn transform_response(
|
||||
&self,
|
||||
model: &str,
|
||||
response: ProviderChatResponseData,
|
||||
response: Self::ResponseBody,
|
||||
) -> Result<ChatCompletionsResponse, ChatResponseError>;
|
||||
}
|
||||
|
||||
pub fn unsupported_param(
|
||||
supported: &'static [(&'static str, &'static str)],
|
||||
supported: &'static [&'static str],
|
||||
config: &'static [&'static str],
|
||||
optional_params: &Map<String, Value>,
|
||||
) -> Option<Unsupported> {
|
||||
|
|
@ -115,9 +110,7 @@ pub fn unsupported_param(
|
|||
.keys()
|
||||
.any(|key| {
|
||||
key != STREAM_PARAM
|
||||
&& !supported
|
||||
.iter()
|
||||
.any(|(_, provider_name)| *provider_name == key)
|
||||
&& !supported.iter().any(|provider_name| *provider_name == key)
|
||||
&& !config.contains(&key.as_str())
|
||||
})
|
||||
.then_some(Unsupported("unrecognized request parameter"))
|
||||
|
|
|
|||
|
|
@ -1,10 +1,9 @@
|
|||
use crate::auth::RequestAuth;
|
||||
use std::time::Duration;
|
||||
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::transformation::{ChatCompletionsAuth, ChatCompletionsProviderConfig};
|
||||
|
||||
/// A `/chat/completions` call as it crosses into the core.
|
||||
///
|
||||
/// `optional_params` arrives already mapped to the provider's own parameter
|
||||
|
|
@ -24,7 +23,7 @@ pub struct ChatCompletionsRequest<'a> {
|
|||
|
||||
pub(super) struct ResolvedChatCompletionsRequest<'a> {
|
||||
pub(super) model: String,
|
||||
pub(super) config: &'static dyn ChatCompletionsProviderConfig,
|
||||
pub(super) config: super::common_utils::ChatProviderConfig,
|
||||
pub(super) messages: Vec<ChatMessage>,
|
||||
pub(super) optional_params: Map<String, Value>,
|
||||
pub(super) api_key: Option<&'a str>,
|
||||
|
|
@ -35,28 +34,16 @@ pub(super) struct ResolvedChatCompletionsRequest<'a> {
|
|||
|
||||
pub(super) struct ProviderChatCompletionsRequest {
|
||||
pub(super) model: String,
|
||||
pub(super) config: &'static dyn ChatCompletionsProviderConfig,
|
||||
pub(super) config: super::common_utils::ChatProviderConfig,
|
||||
pub(super) url: String,
|
||||
pub(super) body: Value,
|
||||
pub(super) upstream_headers: Vec<(String, String)>,
|
||||
pub(super) auth: ChatCompletionsAuth,
|
||||
pub(super) auth: RequestAuth,
|
||||
#[cfg_attr(not(feature = "bedrock-auth"), allow(dead_code))]
|
||||
pub(super) optional_params: Map<String, Value>,
|
||||
pub(super) timeout: Option<Duration>,
|
||||
}
|
||||
|
||||
/// The provider-shaped request body a config produces. Named rather than a bare
|
||||
/// `Value` so the transform contract stays a typed one, mirroring
|
||||
/// [`crate::audio_transcription::types::AudioTranscriptionRequestData`].
|
||||
pub struct ProviderChatRequestData {
|
||||
pub body: Value,
|
||||
}
|
||||
|
||||
/// The raw provider response body handed back to a config for normalization.
|
||||
pub struct ProviderChatResponseData {
|
||||
pub body: Value,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ChatMessageContent {
|
||||
|
|
|
|||
|
|
@ -61,6 +61,25 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool {
|
|||
})
|
||||
}
|
||||
|
||||
pub(crate) fn deserialize_optional_param<'de, D, T>(
|
||||
deserializer: D,
|
||||
) -> Result<Option<Option<T>>, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
T: serde::Deserialize<'de>,
|
||||
{
|
||||
<Option<T> as serde::Deserialize>::deserialize(deserializer).map(Some)
|
||||
}
|
||||
|
||||
pub(crate) fn encode_query_value(value: &str) -> String {
|
||||
const QUERY_VALUE: &percent_encoding::AsciiSet = &percent_encoding::NON_ALPHANUMERIC
|
||||
.remove(b'-')
|
||||
.remove(b'.')
|
||||
.remove(b'_')
|
||||
.remove(b'~');
|
||||
percent_encoding::utf8_percent_encode(value, QUERY_VALUE).to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
use crate::auth::CredentialPlacement;
|
||||
use crate::error::Error;
|
||||
use crate::routing_utils::provider::{CustomLlmProvider, get_custom_llm_provider};
|
||||
|
||||
use super::common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers};
|
||||
use super::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy};
|
||||
use super::transformation::AnthropicMessagesProviderConfig;
|
||||
use super::types::{MessagesRequest, ProviderMessagesRequest};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
|
|
@ -65,16 +66,16 @@ fn validate_environment(
|
|||
) -> Result<Vec<(String, String)>, Error> {
|
||||
let mut headers = string_headers(extra_headers)?;
|
||||
|
||||
let auth_strategy = config.auth_strategy();
|
||||
let already_authorized = has_header(&headers, auth_strategy.header_name())
|
||||
let credential_placement = config.credential_placement();
|
||||
let already_authorized = has_header(&headers, credential_placement.header_name())
|
||||
|| (config.accepts_bearer_auth() && has_bearer_auth(&headers));
|
||||
if !already_authorized {
|
||||
let api_key = config.resolve_api_key(api_key, env_lookup)?;
|
||||
let auth_header = match auth_strategy {
|
||||
MessagesAuthStrategy::Bearer => {
|
||||
let auth_header = match credential_placement {
|
||||
CredentialPlacement::Bearer => {
|
||||
("authorization".to_string(), format!("Bearer {api_key}"))
|
||||
}
|
||||
MessagesAuthStrategy::Header(name) => (name.to_string(), api_key),
|
||||
CredentialPlacement::Header(name) => (name.to_string(), api_key),
|
||||
};
|
||||
headers.push(auth_header);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,37 +1,17 @@
|
|||
use super::types::{AnthropicMessagesRequest, AnthropicMessagesResponse};
|
||||
use crate::Error;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum MessagesAuthStrategy {
|
||||
Bearer,
|
||||
Header(&'static str),
|
||||
}
|
||||
|
||||
impl MessagesAuthStrategy {
|
||||
pub fn header_name(self) -> &'static str {
|
||||
match self {
|
||||
Self::Bearer => "authorization",
|
||||
Self::Header(header_name) => header_name,
|
||||
}
|
||||
}
|
||||
}
|
||||
use crate::auth::CredentialPlacement;
|
||||
|
||||
pub trait AnthropicMessagesProviderConfig: Sync {
|
||||
fn complete_url(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
model: &str,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error>;
|
||||
|
||||
fn resolve_api_key(
|
||||
&self,
|
||||
api_key: Option<&str>,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error>;
|
||||
|
||||
fn auth_strategy(&self) -> MessagesAuthStrategy {
|
||||
MessagesAuthStrategy::Header("x-api-key")
|
||||
fn credential_placement(&self) -> CredentialPlacement {
|
||||
CredentialPlacement::Header("x-api-key")
|
||||
}
|
||||
|
||||
fn accepts_bearer_auth(&self) -> bool {
|
||||
|
|
@ -45,6 +25,13 @@ pub trait AnthropicMessagesProviderConfig: Sync {
|
|||
]
|
||||
}
|
||||
|
||||
fn complete_url(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
model: &str,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error>;
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_request(
|
||||
&self,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use crate::auth::azure::AzureAuthInputs;
|
||||
use crate::constants::{AZURE_AI_OCR_PATH, AZURE_DI_API_VERSION};
|
||||
use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend};
|
||||
use crate::ocr::backends::{BackendConfig, OcrBackend, OcrIntegration, PreparedOcrBackend};
|
||||
use crate::ocr::error::{OcrError, OcrRequestError};
|
||||
use crate::ocr::formats::document_intelligence::{
|
||||
AzureDocumentIntelligenceOcrFormat,
|
||||
|
|
@ -29,7 +29,7 @@ fn encode_model_id(model: &str) -> Result<String, OcrRequestError> {
|
|||
pub struct AzureMistral;
|
||||
|
||||
impl OcrIntegration for AzureMistral {
|
||||
type Host = AzureHost;
|
||||
type Backend = AzureBackend;
|
||||
type Format = MistralOcrFormat;
|
||||
type PreparedDocument = crate::ocr::document::InlineOcrDocument;
|
||||
const FORMAT: Self::Format = MistralOcrFormat;
|
||||
|
|
@ -38,7 +38,7 @@ impl OcrIntegration for AzureMistral {
|
|||
async fn prepare(
|
||||
&self,
|
||||
connection: &OcrConnection,
|
||||
config: &HostConfig<Self>,
|
||||
config: &BackendConfig<Self>,
|
||||
_model: &str,
|
||||
_params: &MistralOcrParams,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
|
|
@ -85,7 +85,7 @@ impl OcrIntegration for AzureMistral {
|
|||
pub struct AzureDocumentIntelligence;
|
||||
|
||||
impl OcrIntegration for AzureDocumentIntelligence {
|
||||
type Host = AzureHost;
|
||||
type Backend = AzureBackend;
|
||||
type Format = AzureDocumentIntelligenceOcrFormat;
|
||||
type PreparedDocument = OcrDocument;
|
||||
const FORMAT: Self::Format = AzureDocumentIntelligenceOcrFormat;
|
||||
|
|
@ -93,7 +93,7 @@ impl OcrIntegration for AzureDocumentIntelligence {
|
|||
async fn prepare(
|
||||
&self,
|
||||
connection: &OcrConnection,
|
||||
config: &HostConfig<Self>,
|
||||
config: &BackendConfig<Self>,
|
||||
model: &str,
|
||||
params: &DocumentIntelligenceParams,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
|
|
@ -166,9 +166,9 @@ impl OcrIntegration for AzureDocumentIntelligence {
|
|||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AzureHost;
|
||||
pub struct AzureBackend;
|
||||
|
||||
impl OcrHost for AzureHost {
|
||||
impl OcrBackend for AzureBackend {
|
||||
type Config = AzureAuthInputs;
|
||||
const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::AzureAi;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use crate::constants::MISTRAL_OCR_API_BASE;
|
||||
use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend};
|
||||
use crate::ocr::backends::{BackendConfig, OcrBackend, OcrIntegration, PreparedOcrBackend};
|
||||
use crate::ocr::error::OcrError;
|
||||
use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams};
|
||||
use crate::ocr::types::{OcrConnection, OcrDocument};
|
||||
|
|
@ -22,7 +22,7 @@ pub fn complete_url(api_base: Option<&str>) -> String {
|
|||
pub struct MistralDirect;
|
||||
|
||||
impl OcrIntegration for MistralDirect {
|
||||
type Host = MistralHost;
|
||||
type Backend = MistralBackend;
|
||||
type Format = MistralOcrFormat;
|
||||
type PreparedDocument = OcrDocument;
|
||||
const FORMAT: Self::Format = MistralOcrFormat;
|
||||
|
|
@ -30,7 +30,7 @@ impl OcrIntegration for MistralDirect {
|
|||
async fn prepare(
|
||||
&self,
|
||||
connection: &OcrConnection,
|
||||
_config: &HostConfig<Self>,
|
||||
_config: &BackendConfig<Self>,
|
||||
_model: &str,
|
||||
_params: &MistralOcrParams,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
|
|
@ -58,9 +58,9 @@ impl OcrIntegration for MistralDirect {
|
|||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct MistralHost;
|
||||
pub struct MistralBackend;
|
||||
|
||||
impl OcrHost for MistralHost {
|
||||
impl OcrBackend for MistralBackend {
|
||||
type Config = ();
|
||||
const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::Mistral;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,17 +18,17 @@ pub struct PreparedOcrBackend {
|
|||
}
|
||||
|
||||
pub trait OcrIntegration: Send + Sync + Sized + 'static {
|
||||
type Host: OcrHost;
|
||||
type Backend: OcrBackend;
|
||||
type Format: OcrFormat;
|
||||
type PreparedDocument: Into<<Self::Format as OcrFormat>::PreparedDocument> + Send;
|
||||
const FORMAT: Self::Format;
|
||||
|
||||
fn provider_name(&self) -> &'static str {
|
||||
Self::Host::PROVIDER.as_str()
|
||||
fn supports_native_request_format(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn format(&self) -> Self::Format {
|
||||
Self::FORMAT
|
||||
fn guard_document_before_preparation(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn decode_input_params(
|
||||
|
|
@ -39,7 +39,7 @@ pub trait OcrIntegration: Send + Sync + Sized + 'static {
|
|||
super::registry::validate_request_format(
|
||||
¶ms,
|
||||
self.supports_native_request_format(),
|
||||
self.provider_name(),
|
||||
Self::Backend::PROVIDER.as_str(),
|
||||
)?;
|
||||
Self::Format::validate_input_params(¶ms)?;
|
||||
super::wire::decode_request_value(serde_json::Value::Object(params), prefix)
|
||||
|
|
@ -48,7 +48,7 @@ pub trait OcrIntegration: Send + Sync + Sized + 'static {
|
|||
fn prepare(
|
||||
&self,
|
||||
connection: &OcrConnection,
|
||||
config: &HostConfig<Self>,
|
||||
config: &BackendConfig<Self>,
|
||||
model: &str,
|
||||
params: &MappedParams<Self>,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
|
|
@ -73,10 +73,6 @@ pub trait OcrIntegration: Send + Sync + Sized + 'static {
|
|||
false
|
||||
}
|
||||
|
||||
fn guard_document_before_preparation(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn read_response(
|
||||
&self,
|
||||
_client: &OcrClient,
|
||||
|
|
@ -90,17 +86,13 @@ pub trait OcrIntegration: Send + Sync + Sized + 'static {
|
|||
> + Send {
|
||||
super::client::read_json_response(response, self.preserve_native_response(params))
|
||||
}
|
||||
|
||||
fn supports_native_request_format(&self) -> bool {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub trait OcrHost: Send + Sync + 'static {
|
||||
pub trait OcrBackend: Send + Sync + 'static {
|
||||
type Config: Clone + std::fmt::Debug + Send + Sync + 'static;
|
||||
const PROVIDER: super::registry::OcrProvider;
|
||||
}
|
||||
|
||||
pub type InputParams<I> = <<I as OcrIntegration>::Format as OcrFormat>::InputParams;
|
||||
pub type MappedParams<I> = <<I as OcrIntegration>::Format as OcrFormat>::MappedParams;
|
||||
pub type HostConfig<I> = <<I as OcrIntegration>::Host as OcrHost>::Config;
|
||||
pub type BackendConfig<I> = <<I as OcrIntegration>::Backend as OcrBackend>::Config;
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use crate::constants::{REDUCTO_ID_PREFIX, REDUCTO_OCR_API_BASE};
|
||||
use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend};
|
||||
use crate::ocr::backends::{BackendConfig, OcrBackend, OcrIntegration, PreparedOcrBackend};
|
||||
use crate::ocr::document::InlineDocument;
|
||||
use crate::ocr::error::{OcrError, OcrRequestError, OcrResponseError};
|
||||
use crate::ocr::formats::reducto::{
|
||||
|
|
@ -10,9 +10,9 @@ use crate::ocr::types::{OcrConnection, OcrDocument};
|
|||
use crate::providers::reducto::auth;
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct ReductoHost;
|
||||
pub struct ReductoBackend;
|
||||
|
||||
impl OcrHost for ReductoHost {
|
||||
impl OcrBackend for ReductoBackend {
|
||||
type Config = ();
|
||||
const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::Reducto;
|
||||
}
|
||||
|
|
@ -78,7 +78,7 @@ macro_rules! impl_reducto_backend {
|
|||
pub struct $integration;
|
||||
|
||||
impl OcrIntegration for $integration {
|
||||
type Host = ReductoHost;
|
||||
type Backend = ReductoBackend;
|
||||
type Format = $format;
|
||||
type PreparedDocument = ReductoFileId;
|
||||
const FORMAT: Self::Format = $format;
|
||||
|
|
@ -86,7 +86,7 @@ macro_rules! impl_reducto_backend {
|
|||
async fn prepare(
|
||||
&self,
|
||||
connection: &OcrConnection,
|
||||
_config: &HostConfig<Self>,
|
||||
_config: &BackendConfig<Self>,
|
||||
_model: &str,
|
||||
_params: &$params,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use crate::constants::{VERTEX_DEEPSEEK_API_BASE, VERTEX_OCR_DEFAULT_LOCATION};
|
||||
use crate::ocr::backends::{HostConfig, OcrHost, OcrIntegration, PreparedOcrBackend};
|
||||
use crate::ocr::backends::{BackendConfig, OcrBackend, OcrIntegration, PreparedOcrBackend};
|
||||
use crate::ocr::error::OcrError;
|
||||
use crate::ocr::formats::deepseek::{DeepSeekOcrFormat, types::DeepSeekOcrParams};
|
||||
use crate::ocr::formats::mistral::{MistralOcrFormat, types::MistralOcrParams};
|
||||
|
|
@ -30,7 +30,7 @@ fn location(config: &VertexAuthInputs, env_lookup: &dyn Fn(&str) -> Option<Strin
|
|||
pub struct VertexMistral;
|
||||
|
||||
impl OcrIntegration for VertexMistral {
|
||||
type Host = VertexHost;
|
||||
type Backend = VertexBackend;
|
||||
type Format = MistralOcrFormat;
|
||||
type PreparedDocument = crate::ocr::document::InlineOcrDocument;
|
||||
const FORMAT: Self::Format = MistralOcrFormat;
|
||||
|
|
@ -39,7 +39,7 @@ impl OcrIntegration for VertexMistral {
|
|||
async fn prepare(
|
||||
&self,
|
||||
connection: &OcrConnection,
|
||||
config: &HostConfig<Self>,
|
||||
config: &BackendConfig<Self>,
|
||||
model: &str,
|
||||
_params: &MistralOcrParams,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
|
|
@ -89,7 +89,7 @@ impl OcrIntegration for VertexMistral {
|
|||
pub struct VertexDeepSeek;
|
||||
|
||||
impl OcrIntegration for VertexDeepSeek {
|
||||
type Host = VertexHost;
|
||||
type Backend = VertexBackend;
|
||||
type Format = DeepSeekOcrFormat;
|
||||
type PreparedDocument = OcrDocument;
|
||||
const FORMAT: Self::Format = DeepSeekOcrFormat;
|
||||
|
|
@ -97,7 +97,7 @@ impl OcrIntegration for VertexDeepSeek {
|
|||
async fn prepare(
|
||||
&self,
|
||||
connection: &OcrConnection,
|
||||
config: &HostConfig<Self>,
|
||||
config: &BackendConfig<Self>,
|
||||
_model: &str,
|
||||
_params: &DeepSeekOcrParams,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
|
|
@ -130,9 +130,9 @@ impl OcrIntegration for VertexDeepSeek {
|
|||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct VertexHost;
|
||||
pub struct VertexBackend;
|
||||
|
||||
impl OcrHost for VertexHost {
|
||||
impl OcrBackend for VertexBackend {
|
||||
type Config = VertexAuthInputs;
|
||||
const PROVIDER: crate::ocr::registry::OcrProvider = crate::ocr::registry::OcrProvider::VertexAi;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,15 +28,12 @@ impl OcrFormat for DeepSeekOcrFormat {
|
|||
type RequestBody = DeepSeekOcrRequest;
|
||||
type ResponseBody = DeepSeekOcrResponse;
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn map_ocr_params(
|
||||
&self,
|
||||
params: Self::InputParams,
|
||||
) -> Result<Self::MappedParams, OcrRequestError> {
|
||||
fn map_params(&self, params: Self::InputParams) -> Result<Self::MappedParams, OcrRequestError> {
|
||||
Ok(params)
|
||||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_ocr_request(
|
||||
fn transform_request(
|
||||
&self,
|
||||
model: &str,
|
||||
document: OcrDocument,
|
||||
|
|
@ -61,7 +58,7 @@ impl OcrFormat for DeepSeekOcrFormat {
|
|||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_ocr_response(
|
||||
fn transform_response(
|
||||
&self,
|
||||
model: &str,
|
||||
response: Self::ResponseBody,
|
||||
|
|
|
|||
|
|
@ -173,10 +173,7 @@ impl OcrFormat for AzureDocumentIntelligenceOcrFormat {
|
|||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn map_ocr_params(
|
||||
&self,
|
||||
params: Self::InputParams,
|
||||
) -> Result<Self::MappedParams, OcrRequestError> {
|
||||
fn map_params(&self, params: Self::InputParams) -> Result<Self::MappedParams, OcrRequestError> {
|
||||
Ok(DocumentIntelligenceParams {
|
||||
pages: params
|
||||
.pages
|
||||
|
|
@ -193,7 +190,7 @@ impl OcrFormat for AzureDocumentIntelligenceOcrFormat {
|
|||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_ocr_request(
|
||||
fn transform_request(
|
||||
&self,
|
||||
_model: &str,
|
||||
document: OcrDocument,
|
||||
|
|
@ -213,7 +210,7 @@ impl OcrFormat for AzureDocumentIntelligenceOcrFormat {
|
|||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_ocr_response(
|
||||
fn transform_response(
|
||||
&self,
|
||||
model: &str,
|
||||
response: Self::ResponseBody,
|
||||
|
|
|
|||
|
|
@ -42,18 +42,6 @@ pub struct DocumentIntelligenceParams {
|
|||
pub(crate) request_format: OcrRequestFormat,
|
||||
}
|
||||
|
||||
impl From<DocumentIntelligenceParams> for DocumentIntelligenceInputParams {
|
||||
fn from(params: DocumentIntelligenceParams) -> Self {
|
||||
Self {
|
||||
pages: params.pages.map(|pages| PagesInput::NativeRange(pages.0)),
|
||||
features: params
|
||||
.features
|
||||
.map(|features| FeaturesInput::CommaSeparated(features.0)),
|
||||
req_format: Some(params.request_format),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub enum DocumentIntelligenceRequest {
|
||||
#[serde(rename = "urlSource")]
|
||||
|
|
|
|||
|
|
@ -16,15 +16,12 @@ impl OcrFormat for MistralOcrFormat {
|
|||
type ResponseBody = MistralOcrResponse;
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn map_ocr_params(
|
||||
&self,
|
||||
params: MistralOcrParams,
|
||||
) -> Result<MistralOcrParams, OcrRequestError> {
|
||||
fn map_params(&self, params: MistralOcrParams) -> Result<MistralOcrParams, OcrRequestError> {
|
||||
Ok(params)
|
||||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_ocr_request(
|
||||
fn transform_request(
|
||||
&self,
|
||||
model: &str,
|
||||
document: OcrDocument,
|
||||
|
|
@ -38,7 +35,7 @@ impl OcrFormat for MistralOcrFormat {
|
|||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_ocr_response(
|
||||
fn transform_response(
|
||||
&self,
|
||||
model: &str,
|
||||
response: MistralOcrResponse,
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ pub mod reducto;
|
|||
|
||||
pub trait OcrFormat: Send + Sync + Sized + 'static {
|
||||
type InputParams: std::fmt::Debug + Clone + Serialize + DeserializeOwned + Send + Sync;
|
||||
type MappedParams: Clone + Serialize + Send + Sync + Into<Self::InputParams>;
|
||||
type MappedParams: Clone + Serialize + Send + Sync;
|
||||
type PreparedDocument: Send;
|
||||
type RequestBody: Serialize + DeserializeOwned + Send + Sync;
|
||||
type ResponseBody: DeserializeOwned + Send;
|
||||
|
|
@ -22,17 +22,14 @@ pub trait OcrFormat: Send + Sync + Sized + 'static {
|
|||
Ok(())
|
||||
}
|
||||
|
||||
fn map_ocr_params(
|
||||
&self,
|
||||
params: Self::InputParams,
|
||||
) -> Result<Self::MappedParams, OcrRequestError>;
|
||||
fn transform_ocr_request(
|
||||
fn map_params(&self, params: Self::InputParams) -> Result<Self::MappedParams, OcrRequestError>;
|
||||
fn transform_request(
|
||||
&self,
|
||||
model: &str,
|
||||
document: Self::PreparedDocument,
|
||||
params: &Self::MappedParams,
|
||||
) -> Result<Self::RequestBody, OcrRequestError>;
|
||||
fn transform_ocr_response(
|
||||
fn transform_response(
|
||||
&self,
|
||||
model: &str,
|
||||
response: Self::ResponseBody,
|
||||
|
|
|
|||
|
|
@ -84,15 +84,12 @@ impl OcrFormat for ReductoParseV3Format {
|
|||
type ResponseBody = ReductoResponse;
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn map_ocr_params(
|
||||
&self,
|
||||
params: Self::InputParams,
|
||||
) -> Result<Self::MappedParams, OcrRequestError> {
|
||||
fn map_params(&self, params: Self::InputParams) -> Result<Self::MappedParams, OcrRequestError> {
|
||||
Ok(params)
|
||||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_ocr_request(
|
||||
fn transform_request(
|
||||
&self,
|
||||
_model: &str,
|
||||
document: ReductoFileId,
|
||||
|
|
@ -105,7 +102,7 @@ impl OcrFormat for ReductoParseV3Format {
|
|||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_ocr_response(
|
||||
fn transform_response(
|
||||
&self,
|
||||
model: &str,
|
||||
response: Self::ResponseBody,
|
||||
|
|
@ -125,15 +122,12 @@ impl OcrFormat for ReductoParseLegacyFormat {
|
|||
type ResponseBody = ReductoResponse;
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn map_ocr_params(
|
||||
&self,
|
||||
params: Self::InputParams,
|
||||
) -> Result<Self::MappedParams, OcrRequestError> {
|
||||
fn map_params(&self, params: Self::InputParams) -> Result<Self::MappedParams, OcrRequestError> {
|
||||
Ok(params)
|
||||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_ocr_request(
|
||||
fn transform_request(
|
||||
&self,
|
||||
_model: &str,
|
||||
document: ReductoFileId,
|
||||
|
|
@ -146,7 +140,7 @@ impl OcrFormat for ReductoParseLegacyFormat {
|
|||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_ocr_response(
|
||||
fn transform_response(
|
||||
&self,
|
||||
model: &str,
|
||||
response: Self::ResponseBody,
|
||||
|
|
|
|||
|
|
@ -109,7 +109,7 @@ where
|
|||
I: OcrIntegration,
|
||||
{
|
||||
let backend = &request.integration;
|
||||
let format = &request.integration.format();
|
||||
let format = &I::FORMAT;
|
||||
let prepared_backend = backend
|
||||
.prepare(
|
||||
&request.connection,
|
||||
|
|
@ -138,7 +138,7 @@ where
|
|||
let document = backend
|
||||
.prepare_document(client, document, &request.connection, &headers)
|
||||
.await?;
|
||||
let body = format.transform_ocr_request(&request.model, document.into(), &request.params)?;
|
||||
let body = format.transform_request(&request.model, document.into(), &request.params)?;
|
||||
let body = if !backend.guard_document_before_preparation() && hooks.has_guardrails() {
|
||||
let guarded = hooks
|
||||
.during_call(OcrDuringCallRequest {
|
||||
|
|
@ -220,7 +220,7 @@ async fn send_ocr_call<I: OcrIntegration>(
|
|||
body,
|
||||
} = request;
|
||||
let backend = &integration;
|
||||
let format = integration.format();
|
||||
let format = I::FORMAT;
|
||||
let mut builder = client
|
||||
.provider_http()
|
||||
.post(&url)
|
||||
|
|
@ -235,7 +235,7 @@ async fn send_ocr_call<I: OcrIntegration>(
|
|||
let decoded = backend
|
||||
.read_response(client, response, &url, &headers, &connection, ¶ms)
|
||||
.await?;
|
||||
let response = format.transform_ocr_response(&model, decoded.data, ¶ms)?;
|
||||
let response = format.transform_response(&model, decoded.data, ¶ms)?;
|
||||
Ok(OcrResponseData {
|
||||
provider_native_response: decoded.native,
|
||||
..response
|
||||
|
|
|
|||
|
|
@ -118,7 +118,7 @@ where
|
|||
})?,
|
||||
"guardrail.optional_params",
|
||||
)
|
||||
.and_then(|params| request.integration.format().map_ocr_params(params))?;
|
||||
.and_then(|params| I::FORMAT.map_params(params))?;
|
||||
Ok(Ok(MappedOcrRequest {
|
||||
document: changed.document,
|
||||
params,
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
use super::backends::{HostConfig, InputParams, MappedParams, OcrIntegration};
|
||||
use super::backends::{BackendConfig, InputParams, MappedParams, OcrIntegration};
|
||||
use super::formats::OcrFormat;
|
||||
|
||||
use super::types::{OcrConnection, OcrDocument};
|
||||
|
|
@ -16,7 +16,7 @@ where
|
|||
pub model: String,
|
||||
pub document: OcrDocument,
|
||||
pub params: MappedParams<I>,
|
||||
pub backend_config: HostConfig<I>,
|
||||
pub backend_config: BackendConfig<I>,
|
||||
pub connection: OcrConnection,
|
||||
}
|
||||
|
||||
|
|
@ -26,13 +26,13 @@ pub(crate) fn prepare_ocr_call<I>(
|
|||
model: String,
|
||||
document: OcrDocument,
|
||||
params: InputParams<I>,
|
||||
backend_config: HostConfig<I>,
|
||||
backend_config: BackendConfig<I>,
|
||||
connection: OcrConnection,
|
||||
) -> Result<MappedOcrRequest<I>, Error>
|
||||
where
|
||||
I: OcrIntegration,
|
||||
{
|
||||
let params = integration.format().map_ocr_params(params)?;
|
||||
let params = I::FORMAT.map_params(params)?;
|
||||
Ok(MappedOcrRequest {
|
||||
integration,
|
||||
model,
|
||||
|
|
|
|||
|
|
@ -10,8 +10,8 @@ pub use super::backends::azure_ai::{AzureDocumentIntelligence, AzureMistral};
|
|||
pub use super::backends::mistral::MistralDirect;
|
||||
pub use super::backends::reducto::{ReductoLegacy, ReductoV3};
|
||||
pub use super::backends::vertex_ai::{VertexDeepSeek, VertexMistral};
|
||||
use super::backends::{HostConfig, InputParams};
|
||||
pub use super::backends::{OcrHost, OcrIntegration};
|
||||
use super::backends::{BackendConfig, InputParams};
|
||||
pub use super::backends::{OcrBackend, OcrIntegration};
|
||||
use super::types::OcrRequestFormat;
|
||||
|
||||
pub(crate) fn validate_request_format(
|
||||
|
|
@ -67,7 +67,7 @@ pub(crate) const REDUCTO_LEGACY: ReductoLegacy = ReductoLegacy;
|
|||
#[derive(Clone, Debug)]
|
||||
pub struct OcrIntegrationInput<I: OcrIntegration> {
|
||||
pub params: InputParams<I>,
|
||||
pub backend_config: HostConfig<I>,
|
||||
pub backend_config: BackendConfig<I>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
|
|
|
|||
|
|
@ -1,14 +1,14 @@
|
|||
use crate::auth::RequestAuth;
|
||||
use crate::chat_completions::error::{ChatRequestError, ChatResponseError};
|
||||
use serde_json::{Map, Value, json};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::chat_completions::conversation::{Conversation, build_conversation};
|
||||
use crate::chat_completions::transformation::{
|
||||
ChatCompletionsAuth, ChatCompletionsProviderConfig, Unsupported, unsupported_message,
|
||||
unsupported_param,
|
||||
ChatCompletionsProviderConfig, Unsupported, unsupported_message, unsupported_param,
|
||||
};
|
||||
use crate::chat_completions::types::{
|
||||
ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, ChatMessage,
|
||||
ProviderChatRequestData, ProviderChatResponseData,
|
||||
};
|
||||
use crate::constants::ANTHROPIC_OAUTH_TOKEN_PREFIX;
|
||||
use crate::error::Error;
|
||||
|
|
@ -28,52 +28,100 @@ use crate::chat_completions::response_utils::{finish_reason_for, unix_now, usage
|
|||
/// per-model gate inside `transform_request`, the function this route replaces.
|
||||
/// Forwarding it would send `top_k` to a model that removed sampling params and
|
||||
/// take a 400 after the call, where Python drops it and succeeds.
|
||||
const SUPPORTED_PARAMS: &[(&str, &str)] = &[
|
||||
("max_tokens", "max_tokens"),
|
||||
("temperature", "temperature"),
|
||||
("top_p", "top_p"),
|
||||
("stop", "stop_sequences"),
|
||||
];
|
||||
const SUPPORTED_PARAMS: &[&str] = &["max_tokens", "temperature", "top_p", "stop_sequences"];
|
||||
|
||||
pub struct AnthropicChatCompletionsConfig;
|
||||
|
||||
pub const ANTHROPIC_CHAT_COMPLETIONS_CONFIG: AnthropicChatCompletionsConfig =
|
||||
AnthropicChatCompletionsConfig;
|
||||
|
||||
fn text_block(text: &str) -> Value {
|
||||
json!({"type": "text", "text": text})
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct AnthropicMappedParams {
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none",
|
||||
deserialize_with = "crate::http_utils::deserialize_optional_param"
|
||||
)]
|
||||
pub max_tokens: Option<Option<u64>>,
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none",
|
||||
deserialize_with = "crate::http_utils::deserialize_optional_param"
|
||||
)]
|
||||
pub temperature: Option<Option<serde_json::Number>>,
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none",
|
||||
deserialize_with = "crate::http_utils::deserialize_optional_param"
|
||||
)]
|
||||
pub top_p: Option<Option<serde_json::Number>>,
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none",
|
||||
deserialize_with = "crate::http_utils::deserialize_optional_param"
|
||||
)]
|
||||
pub stop_sequences: Option<Option<Vec<String>>>,
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none",
|
||||
deserialize_with = "crate::http_utils::deserialize_optional_param"
|
||||
)]
|
||||
pub stream: Option<Option<bool>>,
|
||||
}
|
||||
|
||||
fn anthropic_body(model: &str, conversation: &Conversation, params: Map<String, Value>) -> Value {
|
||||
let messages: Vec<Value> = conversation
|
||||
.turns
|
||||
.iter()
|
||||
.map(|turn| {
|
||||
json!({
|
||||
"role": turn.role.as_str(),
|
||||
"content": turn.texts.iter().map(|text| text_block(text)).collect::<Vec<_>>(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct AnthropicChatRequest {
|
||||
pub model: String,
|
||||
pub messages: Vec<AnthropicTextMessage>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
pub system: Vec<AnthropicTextBlock>,
|
||||
#[serde(flatten)]
|
||||
pub params: AnthropicMappedParams,
|
||||
}
|
||||
|
||||
let system: Vec<Value> = conversation.system.iter().map(|s| text_block(s)).collect();
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct AnthropicTextMessage {
|
||||
pub role: crate::chat_completions::conversation::TurnRole,
|
||||
pub content: Vec<AnthropicTextBlock>,
|
||||
}
|
||||
|
||||
let body = Map::from_iter(
|
||||
[
|
||||
("model".to_string(), json!(model)),
|
||||
("messages".to_string(), json!(messages)),
|
||||
]
|
||||
.into_iter()
|
||||
// Python builds `{"model", "messages", **optional_params}` with
|
||||
// `system` already folded into optional_params, so a caller-supplied
|
||||
// key of the same name wins here too.
|
||||
.chain((!system.is_empty()).then(|| ("system".to_string(), json!(system))))
|
||||
.chain(params),
|
||||
);
|
||||
Value::Object(body)
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum AnthropicTextBlock {
|
||||
Text { text: String },
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct AnthropicChatResponse {
|
||||
pub model: Option<String>,
|
||||
pub content: Option<Vec<AnthropicResponseBlock>>,
|
||||
pub stop_reason: Option<String>,
|
||||
pub usage: Option<AnthropicChatUsage>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(tag = "type", rename_all = "snake_case")]
|
||||
pub enum AnthropicResponseBlock {
|
||||
Text {
|
||||
text: Option<String>,
|
||||
},
|
||||
#[serde(other)]
|
||||
Other,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
pub struct AnthropicChatUsage {
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub cache_read_input_tokens: Option<u64>,
|
||||
pub cache_creation_input_tokens: Option<u64>,
|
||||
}
|
||||
|
||||
impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig {
|
||||
type MappedParams = AnthropicMappedParams;
|
||||
type RequestBody = AnthropicChatRequest;
|
||||
type ResponseBody = AnthropicChatResponse;
|
||||
|
||||
fn complete_url(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
|
|
@ -90,8 +138,8 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig {
|
|||
_model: &str,
|
||||
_optional_params: &Map<String, Value>,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ChatCompletionsAuth, Error> {
|
||||
Ok(ChatCompletionsAuth::Header {
|
||||
) -> Result<RequestAuth, Error> {
|
||||
Ok(RequestAuth::Header {
|
||||
name: "x-api-key",
|
||||
value: resolve_anthropic_api_key(api_key, env_lookup)?,
|
||||
})
|
||||
|
|
@ -119,16 +167,16 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig {
|
|||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)] {
|
||||
fn supported_provider_params(&self) -> &'static [&'static str] {
|
||||
SUPPORTED_PARAMS
|
||||
}
|
||||
|
||||
fn unsupported_reason(
|
||||
fn decline_reason(
|
||||
&self,
|
||||
messages: &[ChatMessage],
|
||||
optional_params: &Map<String, Value>,
|
||||
) -> Option<Unsupported> {
|
||||
unsupported_param(self.supported_openai_params(), &[], optional_params)
|
||||
unsupported_param(self.supported_provider_params(), &[], optional_params)
|
||||
.or_else(|| messages.iter().find_map(unsupported_message))
|
||||
// Anthropic rejects a request whose first turn is not a user turn.
|
||||
// Python only repairs that under `litellm.modify_params`, which the
|
||||
|
|
@ -143,11 +191,29 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig {
|
|||
fn transform_request(
|
||||
&self,
|
||||
model: &str,
|
||||
messages: Vec<ChatMessage>,
|
||||
optional_params: Map<String, Value>,
|
||||
) -> Result<ProviderChatRequestData, ChatRequestError> {
|
||||
Ok(ProviderChatRequestData {
|
||||
body: anthropic_body(model, &build_conversation(&messages), optional_params),
|
||||
conversation: Conversation,
|
||||
params: Self::MappedParams,
|
||||
) -> Result<Self::RequestBody, ChatRequestError> {
|
||||
Ok(AnthropicChatRequest {
|
||||
model: model.to_string(),
|
||||
messages: conversation
|
||||
.turns
|
||||
.into_iter()
|
||||
.map(|turn| AnthropicTextMessage {
|
||||
role: turn.role,
|
||||
content: turn
|
||||
.texts
|
||||
.into_iter()
|
||||
.map(|text| AnthropicTextBlock::Text { text })
|
||||
.collect(),
|
||||
})
|
||||
.collect(),
|
||||
system: conversation
|
||||
.system
|
||||
.into_iter()
|
||||
.map(|text| AnthropicTextBlock::Text { text })
|
||||
.collect(),
|
||||
params,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -155,59 +221,41 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig {
|
|||
fn transform_response(
|
||||
&self,
|
||||
_model: &str,
|
||||
response: ProviderChatResponseData,
|
||||
response: Self::ResponseBody,
|
||||
) -> Result<ChatCompletionsResponse, ChatResponseError> {
|
||||
let body = response
|
||||
.body
|
||||
.as_object()
|
||||
.ok_or(ChatResponseError::NotObject { api: "messages" })?;
|
||||
|
||||
let content = body
|
||||
.get("content")
|
||||
.and_then(Value::as_array)
|
||||
let content = response
|
||||
.content
|
||||
.ok_or(ChatResponseError::MissingField("content"))?;
|
||||
if content
|
||||
.iter()
|
||||
.any(|block| block.get("type").and_then(Value::as_str) != Some("text"))
|
||||
{
|
||||
return Err(ChatResponseError::NonTextContent);
|
||||
}
|
||||
let text: String = content
|
||||
.iter()
|
||||
.filter_map(|block| block.get("text").and_then(Value::as_str))
|
||||
.collect();
|
||||
|
||||
let usage = body
|
||||
.get("usage")
|
||||
.and_then(Value::as_object)
|
||||
let text = content
|
||||
.into_iter()
|
||||
.map(|block| match block {
|
||||
AnthropicResponseBlock::Text { text } => Ok(text.unwrap_or_default()),
|
||||
AnthropicResponseBlock::Other => Err(ChatResponseError::NonTextContent),
|
||||
})
|
||||
.collect::<Result<String, _>>()?;
|
||||
let usage = response
|
||||
.usage
|
||||
.ok_or(ChatResponseError::MissingField("usage"))?;
|
||||
let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0);
|
||||
|
||||
Ok(ChatCompletionsResponse {
|
||||
created: unix_now(),
|
||||
model: body
|
||||
.get("model")
|
||||
.and_then(Value::as_str)
|
||||
.ok_or(ChatResponseError::MissingField("model"))?
|
||||
.to_string(),
|
||||
model: response
|
||||
.model
|
||||
.ok_or(ChatResponseError::MissingField("model"))?,
|
||||
choices: vec![ChatCompletionsChoice {
|
||||
index: 0,
|
||||
message: ChatCompletionsChoiceMessage {
|
||||
role: "assistant".to_string(),
|
||||
content: (!text.is_empty()).then_some(text),
|
||||
},
|
||||
finish_reason: finish_reason_for(
|
||||
body.get("stop_reason")
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or(""),
|
||||
)
|
||||
.to_string(),
|
||||
finish_reason: finish_reason_for(response.stop_reason.as_deref().unwrap_or(""))
|
||||
.to_string(),
|
||||
}],
|
||||
usage: usage_from_parts(
|
||||
field("input_tokens"),
|
||||
field("output_tokens"),
|
||||
field("cache_read_input_tokens"),
|
||||
field("cache_creation_input_tokens"),
|
||||
usage.input_tokens.unwrap_or(0),
|
||||
usage.output_tokens.unwrap_or(0),
|
||||
usage.cache_read_input_tokens.unwrap_or(0),
|
||||
usage.cache_creation_input_tokens.unwrap_or(0),
|
||||
),
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use crate::auth::CredentialPlacement;
|
||||
use crate::auth::error::MissingCredential;
|
||||
use crate::error::{AuthError, Error};
|
||||
use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy};
|
||||
use crate::messages::transformation::AnthropicMessagesProviderConfig;
|
||||
|
||||
const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY";
|
||||
const ANTHROPIC_API_BASE_ENV: &str = "ANTHROPIC_API_BASE";
|
||||
|
|
@ -62,8 +63,8 @@ impl AnthropicMessagesProviderConfig for AnthropicMessagesConfig {
|
|||
resolve_anthropic_api_key(api_key, env_lookup)
|
||||
}
|
||||
|
||||
fn auth_strategy(&self) -> MessagesAuthStrategy {
|
||||
MessagesAuthStrategy::Header("x-api-key")
|
||||
fn credential_placement(&self) -> CredentialPlacement {
|
||||
CredentialPlacement::Header("x-api-key")
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -126,7 +127,9 @@ mod tests {
|
|||
#[test]
|
||||
fn auth_strategy_and_default_headers_match_anthropic() {
|
||||
assert_eq!(
|
||||
ANTHROPIC_MESSAGES_CONFIG.auth_strategy().header_name(),
|
||||
ANTHROPIC_MESSAGES_CONFIG
|
||||
.credential_placement()
|
||||
.header_name(),
|
||||
"x-api-key"
|
||||
);
|
||||
assert_eq!(
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use crate::auth::CredentialPlacement;
|
||||
use crate::auth::error::MissingCredential;
|
||||
use crate::error::{AuthError, Error};
|
||||
use crate::messages::transformation::{AnthropicMessagesProviderConfig, MessagesAuthStrategy};
|
||||
use crate::messages::transformation::AnthropicMessagesProviderConfig;
|
||||
use crate::messages::types::{
|
||||
AnthropicMessage, AnthropicMessagesRequest, AnthropicMessagesResponse, ContentBlock,
|
||||
MessageContent, SystemPrompt,
|
||||
|
|
@ -154,8 +155,8 @@ impl AnthropicMessagesProviderConfig for AzureAnthropicMessagesConfig {
|
|||
resolve_azure_api_key(api_key, env_lookup)
|
||||
}
|
||||
|
||||
fn auth_strategy(&self) -> MessagesAuthStrategy {
|
||||
self.anthropic.auth_strategy()
|
||||
fn credential_placement(&self) -> CredentialPlacement {
|
||||
self.anthropic.credential_placement()
|
||||
}
|
||||
|
||||
fn accepts_bearer_auth(&self) -> bool {
|
||||
|
|
@ -287,7 +288,7 @@ mod tests {
|
|||
fn auth_strategy_is_x_api_key() {
|
||||
assert_eq!(
|
||||
AZURE_ANTHROPIC_MESSAGES_CONFIG
|
||||
.auth_strategy()
|
||||
.credential_placement()
|
||||
.header_name(),
|
||||
"x-api-key"
|
||||
);
|
||||
|
|
|
|||
|
|
@ -1,110 +1,87 @@
|
|||
use serde_json::{Map, Value, json};
|
||||
use super::converse::{
|
||||
ConverseAudio, ConverseAudioSource, ConverseContent, ConverseMappedParams, ConverseMessage,
|
||||
ConverseRequest, ConverseResponseContent, ConverseText,
|
||||
};
|
||||
use crate::auth::RequestAuth;
|
||||
use crate::chat_completions::conversation::TurnRole;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::audio_transcription::transformation::{
|
||||
AudioTranscriptionAuth, AudioTranscriptionProviderConfig,
|
||||
};
|
||||
use crate::audio_transcription::transformation::AudioTranscriptionProviderConfig;
|
||||
use crate::audio_transcription::types::{
|
||||
AudioTranscriptionRequestData, AudioTranscriptionResponseData,
|
||||
AudioTranscriptionRequestData, AudioTranscriptionResponseData, ProviderTranscriptionResponse,
|
||||
TranscriptionAudio, TranscriptionParams,
|
||||
};
|
||||
use crate::error::{Error, json_type_name};
|
||||
use crate::error::Error;
|
||||
|
||||
pub use super::aws_base::{aws_auth_config, bedrock_model_id_and_region, resolve_bedrock_region};
|
||||
use super::constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE};
|
||||
|
||||
const SUPPORTED_PARAMS: &[&str] = &["language", "prompt", "temperature", "response_format"];
|
||||
use super::constants::BEDROCK_RUNTIME_ENDPOINT_TEMPLATE;
|
||||
|
||||
pub static BEDROCK_AUDIO_TRANSCRIPTION_CONFIG: BedrockAudioTranscriptionConfig =
|
||||
BedrockAudioTranscriptionConfig;
|
||||
|
||||
pub struct BedrockAudioTranscriptionConfig;
|
||||
|
||||
fn audio_fields(audio: Value) -> Result<(String, String), Error> {
|
||||
let object = audio.as_object().ok_or_else(|| Error::InvalidType {
|
||||
expected: "object",
|
||||
actual: json_type_name(&audio),
|
||||
})?;
|
||||
let data = object
|
||||
.get("data")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| !value.is_empty())
|
||||
.ok_or(Error::MissingField("audio.data"))?;
|
||||
let format = object
|
||||
.get("format")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| matches!(*value, "wav" | "mp3" | "flac" | "ogg"))
|
||||
.ok_or_else(|| {
|
||||
Error::InvalidRequest("audio.format must be wav, mp3, flac, or ogg".to_string())
|
||||
})?;
|
||||
Ok((data.to_string(), format.to_string()))
|
||||
}
|
||||
|
||||
fn optional_string<'a>(params: &'a Map<String, Value>, key: &str) -> Option<&'a str> {
|
||||
params
|
||||
.get(key)
|
||||
.and_then(Value::as_str)
|
||||
.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig {
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn supported_transcription_params(&self) -> &'static [&'static str] {
|
||||
SUPPORTED_PARAMS
|
||||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_transcription_request(
|
||||
fn transform_request(
|
||||
&self,
|
||||
_model: &str,
|
||||
audio: Value,
|
||||
optional_params: Map<String, Value>,
|
||||
audio: TranscriptionAudio,
|
||||
params: TranscriptionParams,
|
||||
) -> Result<AudioTranscriptionRequestData, Error> {
|
||||
let (data, format) = audio_fields(audio)?;
|
||||
if audio.data.is_empty() {
|
||||
return Err(Error::MissingField("audio.data"));
|
||||
}
|
||||
let mut instruction = "Transcribe the audio. Respond with only the transcript.".to_string();
|
||||
if let Some(language) = optional_string(&optional_params, "language") {
|
||||
if let Some(language) = params.language.as_deref().filter(|value| !value.is_empty()) {
|
||||
instruction.push_str(&format!(" The audio language is {language}."));
|
||||
}
|
||||
if let Some(prompt) = optional_string(&optional_params, "prompt") {
|
||||
if let Some(prompt) = params.prompt.as_deref().filter(|value| !value.is_empty()) {
|
||||
instruction.push_str(&format!(" Additional context: {prompt}"));
|
||||
}
|
||||
let mut inference_config = Map::from_iter([("maxTokens".to_string(), json!(4096))]);
|
||||
if let Some(temperature) = optional_params.get("temperature") {
|
||||
inference_config.insert("temperature".to_string(), temperature.clone());
|
||||
}
|
||||
Ok(AudioTranscriptionRequestData {
|
||||
body: json!({
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"audio": {"format": format, "source": {"bytes": data}}},
|
||||
{"text": instruction}
|
||||
]
|
||||
}],
|
||||
"system": [{"text": "You are a transcription assistant."}],
|
||||
"inferenceConfig": inference_config,
|
||||
}),
|
||||
})
|
||||
Ok(AudioTranscriptionRequestData::Bedrock(ConverseRequest {
|
||||
messages: vec![ConverseMessage {
|
||||
role: TurnRole::User,
|
||||
content: vec![
|
||||
ConverseContent::Audio(ConverseAudio {
|
||||
format: audio.format,
|
||||
source: ConverseAudioSource { bytes: audio.data },
|
||||
}),
|
||||
ConverseContent::Text(instruction),
|
||||
],
|
||||
}],
|
||||
system: vec![ConverseText {
|
||||
text: "You are a transcription assistant.".into(),
|
||||
}],
|
||||
inference_config: ConverseMappedParams {
|
||||
max_tokens: Some(Some(4096)),
|
||||
temperature: params.temperature,
|
||||
..Default::default()
|
||||
},
|
||||
}))
|
||||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn transform_transcription_response(
|
||||
fn transform_response(
|
||||
&self,
|
||||
_model: &str,
|
||||
response_json: Value,
|
||||
response: ProviderTranscriptionResponse,
|
||||
) -> Result<AudioTranscriptionResponseData, Error> {
|
||||
let content = response_json
|
||||
.get("output")
|
||||
.and_then(|value| value.get("message"))
|
||||
.and_then(|value| value.get("content"))
|
||||
.and_then(Value::as_array)
|
||||
let ProviderTranscriptionResponse::Bedrock(response) = response;
|
||||
let content = response
|
||||
.output
|
||||
.and_then(|output| output.message)
|
||||
.and_then(|message| message.content)
|
||||
.ok_or_else(|| {
|
||||
Error::InvalidResponse("Bedrock response has no output content".to_string())
|
||||
Error::InvalidResponse("Bedrock response has no output content".into())
|
||||
})?;
|
||||
let mut text = String::new();
|
||||
for block in content {
|
||||
if let Some(value) = block.get("text").and_then(Value::as_str) {
|
||||
text.push_str(value);
|
||||
}
|
||||
}
|
||||
let text = content
|
||||
.into_iter()
|
||||
.filter_map(|block| match block {
|
||||
ConverseResponseContent::Object(block) => block.text.flatten(),
|
||||
ConverseResponseContent::Other(_) => None,
|
||||
})
|
||||
.collect();
|
||||
Ok(AudioTranscriptionResponseData { text })
|
||||
}
|
||||
|
||||
|
|
@ -131,16 +108,16 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig {
|
|||
))
|
||||
}
|
||||
|
||||
fn auth_strategy(
|
||||
fn auth(
|
||||
&self,
|
||||
_api_key: Option<&str>,
|
||||
model: &str,
|
||||
optional_params: &Map<String, Value>,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<AudioTranscriptionAuth, Error> {
|
||||
) -> Result<RequestAuth, Error> {
|
||||
let (_, model_region) = bedrock_model_id_and_region(model);
|
||||
Ok(AudioTranscriptionAuth::AwsSigV4 {
|
||||
Ok(RequestAuth::AwsSigV4 {
|
||||
region: resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup),
|
||||
service: BEDROCK_SERVICE,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -148,6 +125,7 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn no_env(_: &str) -> Option<String> {
|
||||
None
|
||||
|
|
@ -161,16 +139,19 @@ mod tests {
|
|||
("temperature".to_string(), json!(0)),
|
||||
("timestamp_granularities".to_string(), json!(["word"])),
|
||||
]);
|
||||
let params = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.map_transcription_params(¶ms);
|
||||
let params = serde_json::from_value(Value::Object(params)).unwrap();
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
.transform_transcription_request(
|
||||
.transform_request(
|
||||
"mistral.voxtral-mini-3b-2507",
|
||||
json!({"data": "AQI=", "format": "wav", "filename": "sample.wav"}),
|
||||
serde_json::from_value(
|
||||
json!({"data": "AQI=", "format": "wav", "filename": "sample.wav"}),
|
||||
)
|
||||
.unwrap(),
|
||||
params,
|
||||
)
|
||||
.expect("request");
|
||||
assert_eq!(
|
||||
result.body,
|
||||
serde_json::to_value(result).unwrap(),
|
||||
json!({
|
||||
"messages": [{
|
||||
"role": "user",
|
||||
|
|
@ -188,9 +169,9 @@ mod tests {
|
|||
#[test]
|
||||
fn response_concatenates_content_blocks() {
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
|
||||
.transform_transcription_response(
|
||||
.transform_response(
|
||||
"model",
|
||||
json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}}),
|
||||
serde_json::from_value(json!({"output": {"message": {"content": [{"text": "hello "}, {"text": "world"}]}}})).unwrap(),
|
||||
)
|
||||
.expect("response");
|
||||
assert_eq!(result.text, "hello world");
|
||||
|
|
@ -199,10 +180,13 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn invalid_audio_is_rejected() {
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.transform_transcription_request(
|
||||
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.transform_request(
|
||||
"model",
|
||||
json!({"data": "AQI="}),
|
||||
Map::new(),
|
||||
TranscriptionAudio {
|
||||
data: String::new(),
|
||||
format: crate::audio_transcription::types::AudioFormat::Wav,
|
||||
},
|
||||
TranscriptionParams::default(),
|
||||
);
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,16 +1,19 @@
|
|||
use crate::auth::RequestAuth;
|
||||
use crate::chat_completions::error::{ChatRequestError, ChatResponseError};
|
||||
use serde_json::{Map, Value, json};
|
||||
use crate::providers::bedrock::converse::{
|
||||
ConverseContent, ConverseMappedParams, ConverseMessage, ConverseRequest, ConverseResponse,
|
||||
ConverseResponseContent, ConverseText,
|
||||
};
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::chat_completions::conversation::{Conversation, TurnRole, build_conversation};
|
||||
use crate::chat_completions::response_utils::{finish_reason_for, unix_now, usage_from_parts};
|
||||
use crate::chat_completions::transformation::{
|
||||
ChatCompletionsAuth, ChatCompletionsProviderConfig, Unsupported, unsupported_message,
|
||||
unsupported_param,
|
||||
ChatCompletionsProviderConfig, Unsupported, unsupported_message, unsupported_param,
|
||||
};
|
||||
use crate::chat_completions::types::{
|
||||
ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse,
|
||||
ChatCompletionsUsage, ChatMessage, ChatMessageContent, ProviderChatRequestData,
|
||||
ProviderChatResponseData,
|
||||
ChatCompletionsUsage, ChatMessage, ChatMessageContent,
|
||||
};
|
||||
use crate::error::Error;
|
||||
|
||||
|
|
@ -24,12 +27,7 @@ use super::super::constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT
|
|||
/// `additionalModelRequestFields` for Anthropic base models and to
|
||||
/// `inferenceConfig` otherwise, and that branch reads the model catalog the
|
||||
/// core cannot see.
|
||||
const SUPPORTED_PARAMS: &[(&str, &str)] = &[
|
||||
("max_tokens", "maxTokens"),
|
||||
("temperature", "temperature"),
|
||||
("top_p", "topP"),
|
||||
("stop", "stopSequences"),
|
||||
];
|
||||
const SUPPORTED_PARAMS: &[&str] = &["maxTokens", "temperature", "topP", "stopSequences"];
|
||||
|
||||
const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = "aws_bedrock_runtime_endpoint";
|
||||
|
||||
|
|
@ -56,43 +54,6 @@ pub struct BedrockChatCompletionsConfig;
|
|||
pub const BEDROCK_CHAT_COMPLETIONS_CONFIG: BedrockChatCompletionsConfig =
|
||||
BedrockChatCompletionsConfig;
|
||||
|
||||
fn converse_body(conversation: &Conversation, params: &Map<String, Value>) -> Value {
|
||||
let messages: Vec<Value> = conversation
|
||||
.turns
|
||||
.iter()
|
||||
.map(|turn| {
|
||||
json!({
|
||||
"role": turn.role.as_str(),
|
||||
"content": turn.texts.iter().map(|text| json!({"text": text})).collect::<Vec<_>>(),
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
|
||||
let inference_config = Map::from_iter(SUPPORTED_PARAMS.iter().filter_map(|(_, name)| {
|
||||
params
|
||||
.get(*name)
|
||||
.map(|value| ((*name).to_string(), value.clone()))
|
||||
}));
|
||||
|
||||
let system: Vec<Value> = conversation
|
||||
.system
|
||||
.iter()
|
||||
.map(|text| json!({"text": text}))
|
||||
.collect();
|
||||
|
||||
Value::Object(Map::from_iter(
|
||||
[
|
||||
(
|
||||
"inferenceConfig".to_string(),
|
||||
Value::Object(inference_config),
|
||||
),
|
||||
("messages".to_string(), json!(messages)),
|
||||
]
|
||||
.into_iter()
|
||||
.chain((!system.is_empty()).then(|| ("system".to_string(), json!(system)))),
|
||||
))
|
||||
}
|
||||
|
||||
fn has_blank_text(message: &ChatMessage) -> bool {
|
||||
match &message.content {
|
||||
None => false,
|
||||
|
|
@ -106,6 +67,10 @@ fn has_blank_text(message: &ChatMessage) -> bool {
|
|||
}
|
||||
|
||||
impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig {
|
||||
type MappedParams = ConverseMappedParams;
|
||||
type RequestBody = ConverseRequest;
|
||||
type ResponseBody = ConverseResponse;
|
||||
|
||||
fn complete_url(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
|
|
@ -139,7 +104,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig {
|
|||
model: &str,
|
||||
optional_params: &Map<String, Value>,
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ChatCompletionsAuth, Error> {
|
||||
) -> Result<RequestAuth, Error> {
|
||||
// Python reads `api_key` as the Bedrock bearer token and consults the
|
||||
// env only when the caller passed none, so a caller-supplied empty key
|
||||
// falls through to SigV4 without reaching for the environment. An
|
||||
|
|
@ -152,10 +117,10 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig {
|
|||
}
|
||||
.filter(|token| !token.is_empty());
|
||||
if let Some(token) = bearer {
|
||||
return Ok(ChatCompletionsAuth::Bearer { token });
|
||||
return Ok(RequestAuth::Bearer { token });
|
||||
}
|
||||
let (_, model_region) = bedrock_model_id_and_region(model);
|
||||
Ok(ChatCompletionsAuth::AwsSigV4 {
|
||||
Ok(RequestAuth::AwsSigV4 {
|
||||
region: resolve_bedrock_region(model_region.as_deref(), optional_params, env_lookup),
|
||||
})
|
||||
}
|
||||
|
|
@ -165,7 +130,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig {
|
|||
}
|
||||
|
||||
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
|
||||
fn supported_openai_params(&self) -> &'static [(&'static str, &'static str)] {
|
||||
fn supported_provider_params(&self) -> &'static [&'static str] {
|
||||
SUPPORTED_PARAMS
|
||||
}
|
||||
|
||||
|
|
@ -173,13 +138,13 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig {
|
|||
CONFIG_PARAMS
|
||||
}
|
||||
|
||||
fn unsupported_reason(
|
||||
fn decline_reason(
|
||||
&self,
|
||||
messages: &[ChatMessage],
|
||||
optional_params: &Map<String, Value>,
|
||||
) -> Option<Unsupported> {
|
||||
unsupported_param(
|
||||
self.supported_openai_params(),
|
||||
self.supported_provider_params(),
|
||||
CONFIG_PARAMS,
|
||||
optional_params,
|
||||
)
|
||||
|
|
@ -213,52 +178,56 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig {
|
|||
fn transform_request(
|
||||
&self,
|
||||
_model: &str,
|
||||
messages: Vec<ChatMessage>,
|
||||
optional_params: Map<String, Value>,
|
||||
) -> Result<ProviderChatRequestData, ChatRequestError> {
|
||||
Ok(ProviderChatRequestData {
|
||||
body: converse_body(&build_conversation(&messages), &optional_params),
|
||||
conversation: Conversation,
|
||||
params: Self::MappedParams,
|
||||
) -> Result<Self::RequestBody, ChatRequestError> {
|
||||
Ok(ConverseRequest {
|
||||
messages: conversation
|
||||
.turns
|
||||
.into_iter()
|
||||
.map(|turn| ConverseMessage {
|
||||
role: turn.role,
|
||||
content: turn.texts.into_iter().map(ConverseContent::Text).collect(),
|
||||
})
|
||||
.collect(),
|
||||
system: conversation
|
||||
.system
|
||||
.into_iter()
|
||||
.map(|text| ConverseText { text })
|
||||
.collect(),
|
||||
inference_config: params,
|
||||
})
|
||||
}
|
||||
|
||||
fn transform_response(
|
||||
&self,
|
||||
model: &str,
|
||||
response: ProviderChatResponseData,
|
||||
response: Self::ResponseBody,
|
||||
) -> Result<ChatCompletionsResponse, ChatResponseError> {
|
||||
let body = response
|
||||
.body
|
||||
.as_object()
|
||||
.ok_or(ChatResponseError::NotObject { api: "converse" })?;
|
||||
|
||||
let content = body
|
||||
.get("output")
|
||||
.and_then(|output| output.get("message"))
|
||||
.and_then(|message| message.get("content"))
|
||||
.and_then(Value::as_array)
|
||||
let content = response
|
||||
.output
|
||||
.and_then(|output| output.message)
|
||||
.and_then(|message| message.content)
|
||||
.ok_or(ChatResponseError::MissingField("output.message.content"))?;
|
||||
if content.iter().any(|block| {
|
||||
block
|
||||
.as_object()
|
||||
.is_none_or(|block| block.len() != 1 || !block.contains_key("text"))
|
||||
}) {
|
||||
return Err(ChatResponseError::NonTextContent);
|
||||
}
|
||||
let text: String = content
|
||||
.iter()
|
||||
.filter_map(|block| block.get("text").and_then(Value::as_str))
|
||||
.collect();
|
||||
|
||||
let usage = body
|
||||
.get("usage")
|
||||
.and_then(Value::as_object)
|
||||
let text = content
|
||||
.into_iter()
|
||||
.map(|block| match block {
|
||||
ConverseResponseContent::Object(block)
|
||||
if block.text.is_some() && block.other_fields.is_empty() =>
|
||||
{
|
||||
Ok(block.text.flatten().unwrap_or_default())
|
||||
}
|
||||
_ => Err(ChatResponseError::NonTextContent),
|
||||
})
|
||||
.collect::<Result<String, _>>()?;
|
||||
let usage = response
|
||||
.usage
|
||||
.ok_or(ChatResponseError::MissingField("usage"))?;
|
||||
let field = |name: &str| usage.get(name).and_then(Value::as_u64).unwrap_or(0);
|
||||
let computed = usage_from_parts(
|
||||
field("inputTokens"),
|
||||
field("outputTokens"),
|
||||
field("cacheReadInputTokens"),
|
||||
field("cacheWriteInputTokens"),
|
||||
usage.input_tokens.unwrap_or(0),
|
||||
usage.output_tokens.unwrap_or(0),
|
||||
usage.cache_read_input_tokens.unwrap_or(0),
|
||||
usage.cache_write_input_tokens.unwrap_or(0),
|
||||
);
|
||||
// Converse reports `totalTokens` and Python passes it straight through,
|
||||
// where Anthropic has no such field and Python adds the two counts
|
||||
|
|
@ -267,10 +236,7 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig {
|
|||
// raises there rather than reporting a zero; fall back to the computed
|
||||
// total, which is the closest thing to that without failing the call.
|
||||
let usage = ChatCompletionsUsage {
|
||||
total_tokens: usage
|
||||
.get("totalTokens")
|
||||
.and_then(Value::as_u64)
|
||||
.unwrap_or(computed.total_tokens),
|
||||
total_tokens: usage.total_tokens.unwrap_or(computed.total_tokens),
|
||||
..computed
|
||||
};
|
||||
|
||||
|
|
@ -288,10 +254,8 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig {
|
|||
// on this path alone.
|
||||
content: Some(text),
|
||||
},
|
||||
finish_reason: finish_reason_for(
|
||||
body.get("stopReason").and_then(Value::as_str).unwrap_or(""),
|
||||
)
|
||||
.to_string(),
|
||||
finish_reason: finish_reason_for(response.stop_reason.as_deref().unwrap_or(""))
|
||||
.to_string(),
|
||||
}],
|
||||
usage,
|
||||
})
|
||||
|
|
|
|||
116
litellm-rust/crates/core/src/providers/bedrock/converse.rs
Normal file
116
litellm-rust/crates/core/src/providers/bedrock/converse.rs
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::chat_completions::conversation::TurnRole;
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ConverseMappedParams {
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none",
|
||||
deserialize_with = "crate::http_utils::deserialize_optional_param"
|
||||
)]
|
||||
pub max_tokens: Option<Option<u64>>,
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none",
|
||||
deserialize_with = "crate::http_utils::deserialize_optional_param"
|
||||
)]
|
||||
pub temperature: Option<Option<serde_json::Number>>,
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none",
|
||||
deserialize_with = "crate::http_utils::deserialize_optional_param"
|
||||
)]
|
||||
pub top_p: Option<Option<serde_json::Number>>,
|
||||
#[serde(
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none",
|
||||
deserialize_with = "crate::http_utils::deserialize_optional_param"
|
||||
)]
|
||||
pub stop_sequences: Option<Option<Vec<String>>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ConverseRequest {
|
||||
pub messages: Vec<ConverseMessage>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub system: Vec<ConverseText>,
|
||||
pub inference_config: ConverseMappedParams,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ConverseMessage {
|
||||
pub role: TurnRole,
|
||||
pub content: Vec<ConverseContent>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum ConverseContent {
|
||||
Text(String),
|
||||
Audio(ConverseAudio),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ConverseText {
|
||||
pub text: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ConverseAudio {
|
||||
pub format: crate::audio_transcription::types::AudioFormat,
|
||||
pub source: ConverseAudioSource,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct ConverseAudioSource {
|
||||
pub bytes: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct ConverseResponse {
|
||||
pub output: Option<ConverseOutput>,
|
||||
#[serde(rename = "stopReason")]
|
||||
pub stop_reason: Option<String>,
|
||||
pub usage: Option<ConverseUsage>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct ConverseOutput {
|
||||
pub message: Option<ConverseResponseMessage>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct ConverseResponseMessage {
|
||||
pub content: Option<Vec<ConverseResponseContent>>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(untagged)]
|
||||
pub enum ConverseResponseContent {
|
||||
Object(ConverseResponseObject),
|
||||
Other(serde::de::IgnoredAny),
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
pub struct ConverseResponseObject {
|
||||
#[serde(
|
||||
default,
|
||||
deserialize_with = "crate::http_utils::deserialize_optional_param"
|
||||
)]
|
||||
pub text: Option<Option<String>>,
|
||||
#[serde(flatten)]
|
||||
pub other_fields: std::collections::BTreeMap<String, serde::de::IgnoredAny>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ConverseUsage {
|
||||
pub input_tokens: Option<u64>,
|
||||
pub output_tokens: Option<u64>,
|
||||
pub total_tokens: Option<u64>,
|
||||
pub cache_read_input_tokens: Option<u64>,
|
||||
pub cache_write_input_tokens: Option<u64>,
|
||||
}
|
||||
|
|
@ -7,4 +7,5 @@ pub mod audio_transcription;
|
|||
pub mod aws_base;
|
||||
pub mod chat_completions;
|
||||
mod constants;
|
||||
pub mod converse;
|
||||
mod resolved_credentials_cache;
|
||||
|
|
|
|||
|
|
@ -8,23 +8,6 @@ pub const OPENAI_REALTIME_DEFAULT_API_BASE: &str = "https://api.openai.com";
|
|||
/// Path appended to the resolved host base to reach the realtime endpoint.
|
||||
pub const OPENAI_REALTIME_PATH: &str = "/v1/realtime";
|
||||
|
||||
/// Percent-encode a query value, escaping any char outside the RFC 3986
|
||||
/// unreserved set (`A-Za-z0-9-._~`). Keeps us dependency-free; common realtime
|
||||
/// model slugs have no special chars, but this stays correct for the rest.
|
||||
fn percent_encode(value: &str) -> String {
|
||||
let mut encoded = String::with_capacity(value.len());
|
||||
for byte in value.bytes() {
|
||||
let unreserved = byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~');
|
||||
if unreserved {
|
||||
encoded.push(byte as char);
|
||||
} else {
|
||||
encoded.push('%');
|
||||
encoded.push_str(&format!("{byte:02X}"));
|
||||
}
|
||||
}
|
||||
encoded
|
||||
}
|
||||
|
||||
/// Build the realtime WebSocket URL, porting Python's `OpenAIRealtime._construct_url`.
|
||||
///
|
||||
/// Blank/whitespace `api_base` is treated as absent (guard at resolution time),
|
||||
|
|
@ -55,7 +38,7 @@ pub fn complete_url(api_base: Option<&str>, model: &str) -> String {
|
|||
|
||||
format!(
|
||||
"{base}{OPENAI_REALTIME_PATH}?model={}",
|
||||
percent_encode(model)
|
||||
crate::http_utils::encode_query_value(model)
|
||||
)
|
||||
}
|
||||
|
||||
|
|
@ -68,7 +51,7 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig {
|
|||
complete_url(api_base, model)
|
||||
}
|
||||
|
||||
fn transform_realtime_request(
|
||||
fn transform_request(
|
||||
&self,
|
||||
event: &RealtimeEvent,
|
||||
_model: &str,
|
||||
|
|
@ -76,7 +59,7 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig {
|
|||
Ok(RealtimeTransformResult::passthrough(event.clone()))
|
||||
}
|
||||
|
||||
fn transform_realtime_response(
|
||||
fn transform_response(
|
||||
&self,
|
||||
event: &RealtimeEvent,
|
||||
_model: &str,
|
||||
|
|
@ -85,18 +68,18 @@ impl RealtimeProviderConfig for OpenAiRealtimeConfig {
|
|||
}
|
||||
}
|
||||
|
||||
pub fn transform_realtime_request(
|
||||
pub fn transform_request(
|
||||
event: &RealtimeEvent,
|
||||
model: &str,
|
||||
) -> Result<RealtimeTransformResult, Error> {
|
||||
OPENAI_REALTIME_CONFIG.transform_realtime_request(event, model)
|
||||
OPENAI_REALTIME_CONFIG.transform_request(event, model)
|
||||
}
|
||||
|
||||
pub fn transform_realtime_response(
|
||||
pub fn transform_response(
|
||||
event: &RealtimeEvent,
|
||||
model: &str,
|
||||
) -> Result<RealtimeTransformResult, Error> {
|
||||
OPENAI_REALTIME_CONFIG.transform_realtime_response(event, model)
|
||||
OPENAI_REALTIME_CONFIG.transform_response(event, model)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
@ -172,8 +155,7 @@ mod tests {
|
|||
let event: RealtimeEvent =
|
||||
serde_json::from_str(r#"{"type":"session.update","session":{"voice":"alloy"}}"#)
|
||||
.expect("valid event");
|
||||
let result =
|
||||
transform_realtime_request(&event, "gpt-realtime").expect("passthrough is infallible");
|
||||
let result = transform_request(&event, "gpt-realtime").expect("passthrough is infallible");
|
||||
assert_eq!(result.events, vec![event]);
|
||||
}
|
||||
|
||||
|
|
@ -182,8 +164,7 @@ mod tests {
|
|||
let event: RealtimeEvent =
|
||||
serde_json::from_str(r#"{"type":"response.output_audio.delta","delta":"abc=="}"#)
|
||||
.expect("valid event");
|
||||
let result =
|
||||
transform_realtime_response(&event, "gpt-realtime").expect("passthrough is infallible");
|
||||
let result = transform_response(&event, "gpt-realtime").expect("passthrough is infallible");
|
||||
assert_eq!(result.events, vec![event]);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,16 +2,12 @@ use crate::Error;
|
|||
use crate::responses::types::{ResponsesWsEvent, ResponsesWsTransformResult};
|
||||
use crate::responses::websocket::{ResponsesWebSocketProviderConfig, enforce_model};
|
||||
|
||||
pub struct OpenAIResponsesWsConfig;
|
||||
pub struct OpenAiResponsesWsConfig;
|
||||
|
||||
pub const OPENAI_RESPONSES_WS_CONFIG: OpenAIResponsesWsConfig = OpenAIResponsesWsConfig;
|
||||
pub const OPENAI_RESPONSES_WS_CONFIG: OpenAiResponsesWsConfig = OpenAiResponsesWsConfig;
|
||||
|
||||
impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig {
|
||||
fn supports_native_websocket(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn transform_ws_request(
|
||||
impl ResponsesWebSocketProviderConfig for OpenAiResponsesWsConfig {
|
||||
fn transform_request(
|
||||
&self,
|
||||
event: &ResponsesWsEvent,
|
||||
model: &str,
|
||||
|
|
@ -21,7 +17,7 @@ impl ResponsesWebSocketProviderConfig for OpenAIResponsesWsConfig {
|
|||
)))
|
||||
}
|
||||
|
||||
fn transform_ws_response(
|
||||
fn transform_response(
|
||||
&self,
|
||||
event: &ResponsesWsEvent,
|
||||
_model: &str,
|
||||
|
|
@ -40,9 +36,8 @@ mod tests {
|
|||
serde_json::from_value(serde_json::json!({"type":"response.create"}))
|
||||
.expect("valid event");
|
||||
let result = OPENAI_RESPONSES_WS_CONFIG
|
||||
.transform_ws_request(&event, "gpt-5")
|
||||
.transform_request(&event, "gpt-5")
|
||||
.expect("valid transform");
|
||||
assert_eq!(result.events[0].model(), Some("gpt-5"));
|
||||
assert!(OPENAI_RESPONSES_WS_CONFIG.supports_native_websocket());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,20 +1,20 @@
|
|||
use crate::Error;
|
||||
use crate::realtime::types::{RealtimeEvent, RealtimeTransformResult};
|
||||
|
||||
pub trait RealtimeProviderConfig {
|
||||
pub trait RealtimeProviderConfig: Sync {
|
||||
/// Build the upstream WebSocket URL (e.g. `wss://api.openai.com/v1/realtime?model=…`).
|
||||
/// Pure string construction only — no network, no env.
|
||||
fn complete_url(&self, api_base: Option<&str>, model: &str) -> String;
|
||||
|
||||
/// Transform a client → backend event before it is forwarded upstream.
|
||||
fn transform_realtime_request(
|
||||
fn transform_request(
|
||||
&self,
|
||||
event: &RealtimeEvent,
|
||||
model: &str,
|
||||
) -> Result<RealtimeTransformResult, Error>;
|
||||
|
||||
/// Transform a backend → client event before it is forwarded downstream.
|
||||
fn transform_realtime_response(
|
||||
fn transform_response(
|
||||
&self,
|
||||
event: &RealtimeEvent,
|
||||
model: &str,
|
||||
|
|
|
|||
|
|
@ -3,36 +3,28 @@ use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH}
|
|||
use crate::responses::types::{ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult};
|
||||
|
||||
pub trait ResponsesWebSocketProviderConfig: Sync {
|
||||
fn supports_native_websocket(&self) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
fn model_in_websocket_url(&self) -> bool {
|
||||
true
|
||||
}
|
||||
|
||||
fn complete_websocket_url(&self, api_base: Option<&str>, model: &str) -> String {
|
||||
complete_websocket_url(api_base, model, self.model_in_websocket_url())
|
||||
fn complete_url(&self, api_base: Option<&str>, model: &str) -> String {
|
||||
complete_url(api_base, model, self.model_in_websocket_url())
|
||||
}
|
||||
|
||||
fn transform_ws_request(
|
||||
fn transform_request(
|
||||
&self,
|
||||
event: &ResponsesWsEvent,
|
||||
model: &str,
|
||||
) -> Result<ResponsesWsTransformResult, Error>;
|
||||
|
||||
fn transform_ws_response(
|
||||
fn transform_response(
|
||||
&self,
|
||||
event: &ResponsesWsEvent,
|
||||
model: &str,
|
||||
) -> Result<ResponsesWsTransformResult, Error>;
|
||||
}
|
||||
|
||||
pub fn complete_websocket_url(
|
||||
api_base: Option<&str>,
|
||||
model: &str,
|
||||
model_in_websocket_url: bool,
|
||||
) -> String {
|
||||
pub fn complete_url(api_base: Option<&str>, model: &str, model_in_websocket_url: bool) -> String {
|
||||
let base = api_base
|
||||
.map(str::trim)
|
||||
.filter(|value| !value.is_empty())
|
||||
|
|
@ -67,23 +59,10 @@ pub fn complete_websocket_url(
|
|||
format!(
|
||||
"{url}{}model={}",
|
||||
if query.is_some() { "&" } else { "?" },
|
||||
percent_encode(model)
|
||||
crate::http_utils::encode_query_value(model)
|
||||
)
|
||||
}
|
||||
|
||||
fn percent_encode(value: &str) -> String {
|
||||
value
|
||||
.bytes()
|
||||
.map(|byte| {
|
||||
if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') {
|
||||
format!("{}", byte as char)
|
||||
} else {
|
||||
format!("%{byte:02X}")
|
||||
}
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn enforce_model(event: &ResponsesWsEvent, model: &str) -> ResponsesWsEvent {
|
||||
if !event.is_response_create() {
|
||||
return event.clone();
|
||||
|
|
@ -136,19 +115,19 @@ mod tests {
|
|||
#[test]
|
||||
fn url_construction_matches_python_defaults_and_query_behavior() {
|
||||
assert_eq!(
|
||||
complete_websocket_url(None, "gpt-5", true),
|
||||
complete_url(None, "gpt-5", true),
|
||||
"wss://api.openai.com/v1/responses?model=gpt-5"
|
||||
);
|
||||
assert_eq!(
|
||||
complete_websocket_url(Some("http://localhost:8080/"), "gpt 5", true),
|
||||
complete_url(Some("http://localhost:8080/"), "gpt 5", true),
|
||||
"ws://localhost:8080/responses?model=gpt%205"
|
||||
);
|
||||
assert_eq!(
|
||||
complete_websocket_url(Some("https://example.test/v1?foo=bar"), "gpt-5", true),
|
||||
complete_url(Some("https://example.test/v1?foo=bar"), "gpt-5", true),
|
||||
"wss://example.test/v1/responses?foo=bar&model=gpt-5"
|
||||
);
|
||||
assert_eq!(
|
||||
complete_websocket_url(Some("https://example.test?model=existing"), "gpt-5", true),
|
||||
complete_url(Some("https://example.test?model=existing"), "gpt-5", true),
|
||||
"wss://example.test/responses?model=existing"
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,19 +13,29 @@ fn params(value: Value) -> Map<String, Value> {
|
|||
}
|
||||
|
||||
fn transform(model: &str, msgs: Value, opts: Value) -> Value {
|
||||
ANTHROPIC_CHAT_COMPLETIONS_CONFIG
|
||||
.transform_request(model, messages(msgs), params(opts))
|
||||
.expect("request transforms")
|
||||
.body
|
||||
serde_json::to_value(
|
||||
ANTHROPIC_CHAT_COMPLETIONS_CONFIG
|
||||
.transform_request(
|
||||
model,
|
||||
build_conversation(&messages(msgs)),
|
||||
serde_json::from_value(opts).unwrap(),
|
||||
)
|
||||
.expect("request transforms"),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn transform_response(body: Value) -> Result<ChatCompletionsResponse, ChatResponseError> {
|
||||
ANTHROPIC_CHAT_COMPLETIONS_CONFIG
|
||||
.transform_response("claude-sonnet-4-5", ProviderChatResponseData { body })
|
||||
crate::chat_completions::handler::decode_response(
|
||||
&ANTHROPIC_CHAT_COMPLETIONS_CONFIG,
|
||||
"claude-sonnet-4-5",
|
||||
body,
|
||||
"messages",
|
||||
)
|
||||
}
|
||||
|
||||
fn reason(msgs: Value, opts: Value) -> Option<Unsupported> {
|
||||
ANTHROPIC_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts))
|
||||
ANTHROPIC_CHAT_COMPLETIONS_CONFIG.decline_reason(&messages(msgs), ¶ms(opts))
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -426,7 +436,7 @@ fn resolves_the_messages_url_and_x_api_key_auth() {
|
|||
config
|
||||
.auth(Some("sk-x"), "claude-sonnet-4-5", &Map::new(), &|_| None)
|
||||
.expect("auth resolves"),
|
||||
ChatCompletionsAuth::Header {
|
||||
RequestAuth::Header {
|
||||
name: "x-api-key",
|
||||
value: "sk-x".to_string()
|
||||
}
|
||||
|
|
@ -439,3 +449,29 @@ fn resolves_the_messages_url_and_x_api_key_auth() {
|
|||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_explicit_null_mapped_params() {
|
||||
let body = transform(
|
||||
"model",
|
||||
json!([{"role":"user","content":"hi"}]),
|
||||
json!({"max_tokens":null,"temperature":null}),
|
||||
);
|
||||
assert!(body.get("max_tokens").is_some_and(Value::is_null));
|
||||
assert!(body.get("temperature").is_some_and(Value::is_null));
|
||||
assert!(body.get("top_p").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_response_fields_do_not_expose_provider_content() {
|
||||
let error = transform_response(json!({
|
||||
"model": "model",
|
||||
"content": [{"type":"text","text":"private document"}],
|
||||
"usage": {"input_tokens":"private token value"}
|
||||
}))
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
error,
|
||||
ChatResponseError::InvalidJson("invalid provider response fields".into())
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -67,9 +67,8 @@ fn invalid_page_types_return_typed_errors(#[case] pages: Value) {
|
|||
fn invalid_page_values_never_panic(#[case] pages: Value) {
|
||||
let input = serde_json::from_value(json!({"pages":pages})).unwrap();
|
||||
assert!(
|
||||
AZURE_DOCUMENT_INTELLIGENCE
|
||||
.format()
|
||||
.map_ocr_params(input)
|
||||
crate::ocr::registry::AzureDocumentIntelligence::FORMAT
|
||||
.map_params(input)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
|
@ -105,9 +104,8 @@ async fn document_intelligence_maps_features(#[case] features: Value, #[case] ex
|
|||
#[case(json!(""))]
|
||||
fn document_intelligence_rejects_invalid_features(#[case] features: Value) {
|
||||
assert!(
|
||||
AZURE_DOCUMENT_INTELLIGENCE
|
||||
.format()
|
||||
.map_ocr_params(serde_json::from_value(json!({"features":features})).unwrap())
|
||||
crate::ocr::registry::AzureDocumentIntelligence::FORMAT
|
||||
.map_params(serde_json::from_value(json!({"features":features})).unwrap())
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,25 +13,29 @@ fn params(value: Value) -> Map<String, Value> {
|
|||
}
|
||||
|
||||
fn transform(msgs: Value, opts: Value) -> Value {
|
||||
BEDROCK_CHAT_COMPLETIONS_CONFIG
|
||||
.transform_request(
|
||||
"anthropic.claude-sonnet-4-5-v1:0",
|
||||
messages(msgs),
|
||||
params(opts),
|
||||
)
|
||||
.expect("request transforms")
|
||||
.body
|
||||
serde_json::to_value(
|
||||
BEDROCK_CHAT_COMPLETIONS_CONFIG
|
||||
.transform_request(
|
||||
"anthropic.claude-sonnet-4-5-v1:0",
|
||||
build_conversation(&messages(msgs)),
|
||||
serde_json::from_value(opts).unwrap(),
|
||||
)
|
||||
.expect("request transforms"),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn transform_response(body: Value) -> Result<ChatCompletionsResponse, ChatResponseError> {
|
||||
BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response(
|
||||
crate::chat_completions::handler::decode_response(
|
||||
&BEDROCK_CHAT_COMPLETIONS_CONFIG,
|
||||
"anthropic.claude-sonnet-4-5-v1:0",
|
||||
ProviderChatResponseData { body },
|
||||
body,
|
||||
"converse",
|
||||
)
|
||||
}
|
||||
|
||||
fn reason(msgs: Value, opts: Value) -> Option<Unsupported> {
|
||||
BEDROCK_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts))
|
||||
BEDROCK_CHAT_COMPLETIONS_CONFIG.decline_reason(&messages(msgs), ¶ms(opts))
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
@ -279,7 +283,7 @@ fn signs_with_sigv4_in_the_resolved_region() {
|
|||
&|_| None
|
||||
)
|
||||
.expect("auth resolves"),
|
||||
ChatCompletionsAuth::AwsSigV4 {
|
||||
RequestAuth::AwsSigV4 {
|
||||
region: "eu-central-1".to_string()
|
||||
}
|
||||
);
|
||||
|
|
@ -304,10 +308,10 @@ fn a_bearer_token_outranks_sigv4_the_way_python_resolves_it() {
|
|||
)
|
||||
.expect("auth resolves")
|
||||
};
|
||||
let bearer = |token: &str| ChatCompletionsAuth::Bearer {
|
||||
let bearer = |token: &str| RequestAuth::Bearer {
|
||||
token: token.to_string(),
|
||||
};
|
||||
let sigv4 = ChatCompletionsAuth::AwsSigV4 {
|
||||
let sigv4 = RequestAuth::AwsSigV4 {
|
||||
region: "eu-central-1".to_string(),
|
||||
};
|
||||
|
||||
|
|
@ -575,3 +579,40 @@ fn host_supplied_credentials_outrank_ambient_profile_and_role_state() {
|
|||
);
|
||||
assert!(host_supplied_credentials(&Map::new()).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn preserves_explicit_null_mapped_params() {
|
||||
let body = transform(
|
||||
json!([{"role":"user","content":"hi"}]),
|
||||
json!({"maxTokens":null,"temperature":null}),
|
||||
);
|
||||
assert!(
|
||||
body["inferenceConfig"]
|
||||
.get("maxTokens")
|
||||
.is_some_and(Value::is_null)
|
||||
);
|
||||
assert!(
|
||||
body["inferenceConfig"]
|
||||
.get("temperature")
|
||||
.is_some_and(Value::is_null)
|
||||
);
|
||||
assert!(body["inferenceConfig"].get("topP").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_empty_or_mixed_converse_blocks() {
|
||||
for block in [json!({}), json!({"text":"visible","toolUse":{}})] {
|
||||
assert_eq!(
|
||||
transform_response(json!({
|
||||
"output":{"message":{"content":[block]}},"usage":{}
|
||||
}))
|
||||
.unwrap_err(),
|
||||
ChatResponseError::NonTextContent
|
||||
);
|
||||
}
|
||||
let response = transform_response(json!({
|
||||
"output":{"message":{"content":[{"text":null}]}},"usage":{}
|
||||
}))
|
||||
.unwrap();
|
||||
assert_eq!(response.choices[0].message.content.as_deref(), Some(""));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,10 @@
|
|||
use crate::auth::RequestAuth;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::error::Error;
|
||||
|
||||
use super::prepare::{prepare_provider_request, resolve_request};
|
||||
use super::transformation::ChatCompletionsAuth;
|
||||
|
||||
use super::types::{ChatCompletionsRequest, ProviderChatCompletionsRequest};
|
||||
|
||||
fn prepare_chat_completions_call(
|
||||
|
|
@ -89,7 +90,7 @@ fn adds_the_auth_and_default_headers() {
|
|||
);
|
||||
assert!(matches!(
|
||||
prepared.auth,
|
||||
ChatCompletionsAuth::Header {
|
||||
RequestAuth::Header {
|
||||
name: "x-api-key",
|
||||
..
|
||||
}
|
||||
|
|
@ -287,7 +288,7 @@ fn prepares_a_bedrock_call_without_resolving_credentials() {
|
|||
);
|
||||
assert_eq!(
|
||||
prepared.auth,
|
||||
ChatCompletionsAuth::AwsSigV4 {
|
||||
RequestAuth::AwsSigV4 {
|
||||
region: "us-east-1".to_string()
|
||||
}
|
||||
);
|
||||
|
|
@ -468,7 +469,7 @@ fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
|
|||
.expect("prepares");
|
||||
assert_eq!(
|
||||
prepared.auth,
|
||||
ChatCompletionsAuth::Bearer {
|
||||
RequestAuth::Bearer {
|
||||
token: "sk-test".to_string()
|
||||
}
|
||||
);
|
||||
|
|
@ -814,3 +815,36 @@ mod round_trip {
|
|||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_provider_params_decline_before_credentials_are_resolved() {
|
||||
for optional_params in [json!({"max_tokens":"many"}), json!({"stop_sequences":[42]})] {
|
||||
let request = ChatCompletionsRequest {
|
||||
model: "anthropic/model",
|
||||
custom_llm_provider: None,
|
||||
messages: json!([{"role":"user","content":"hi"}]),
|
||||
optional_params: optional_params.as_object().unwrap().clone(),
|
||||
api_key: None,
|
||||
api_base: None,
|
||||
extra_headers: None,
|
||||
timeout: None,
|
||||
};
|
||||
assert_eq!(
|
||||
super::chat_completions_decline_reason(
|
||||
request.model,
|
||||
None,
|
||||
request.messages.clone(),
|
||||
&request.optional_params
|
||||
),
|
||||
Some("invalid provider-mapped parameters"),
|
||||
);
|
||||
let error = match resolve_request(request) {
|
||||
Ok(_) => panic!("malformed params must decline"),
|
||||
Err(error) => error,
|
||||
};
|
||||
assert_eq!(
|
||||
error,
|
||||
Error::Unsupported("invalid provider-mapped parameters")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,13 +25,12 @@ pub(crate) async fn perform_ocr(
|
|||
ocr_client().perform(request).await
|
||||
}
|
||||
|
||||
pub(crate) fn params<I>(integration: &I, value: Value) -> MappedParams<I>
|
||||
pub(crate) fn params<I>(_integration: &I, value: Value) -> MappedParams<I>
|
||||
where
|
||||
I: OcrIntegration,
|
||||
{
|
||||
integration
|
||||
.format()
|
||||
.map_ocr_params(serde_json::from_value(value).unwrap())
|
||||
I::FORMAT
|
||||
.map_params(serde_json::from_value(value).unwrap())
|
||||
.unwrap()
|
||||
}
|
||||
pub(crate) fn transform<I>(
|
||||
|
|
@ -43,16 +42,12 @@ pub(crate) fn transform<I>(
|
|||
where
|
||||
I: OcrIntegration,
|
||||
{
|
||||
let params = integration
|
||||
.format()
|
||||
.map_ocr_params(serde_json::from_value(options).unwrap())?;
|
||||
let params = I::FORMAT.map_params(serde_json::from_value(options).unwrap())?;
|
||||
let decoded = decode_response(
|
||||
&serde_json::to_vec(&value).unwrap(),
|
||||
integration.preserve_native_response(¶ms),
|
||||
)?;
|
||||
let mut response = integration
|
||||
.format()
|
||||
.transform_ocr_response(model, decoded.data, ¶ms)?;
|
||||
let mut response = I::FORMAT.transform_response(model, decoded.data, ¶ms)?;
|
||||
response.provider_native_response = decoded.native;
|
||||
Ok(response.into_json())
|
||||
}
|
||||
|
|
@ -65,9 +60,7 @@ pub(crate) async fn body<I>(
|
|||
where
|
||||
I: OcrIntegration,
|
||||
{
|
||||
let params = integration
|
||||
.format()
|
||||
.map_ocr_params(serde_json::from_value(options).unwrap())?;
|
||||
let params = I::FORMAT.map_params(serde_json::from_value(options).unwrap())?;
|
||||
let client = ocr_client();
|
||||
let document = integration
|
||||
.prepare_document(
|
||||
|
|
@ -78,12 +71,8 @@ where
|
|||
)
|
||||
.await?;
|
||||
Ok(
|
||||
serde_json::to_value(integration.format().transform_ocr_request(
|
||||
model,
|
||||
document.into(),
|
||||
¶ms,
|
||||
)?)
|
||||
.unwrap(),
|
||||
serde_json::to_value(I::FORMAT.transform_request(model, document.into(), ¶ms)?)
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
pub(crate) fn wire_request(model: &str, base: &str, options: Value) -> OcrRequest {
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ COMMON_MAPPINGS: Final = (
|
|||
span="python_provider_supported_openai_params",
|
||||
python_frame=r"AnthropicConfig\.get_supported_openai_params$",
|
||||
),
|
||||
mapping(rust_span="supported_openai_params"),
|
||||
mapping(rust_span="supported_provider_params"),
|
||||
mapping(rust_span="validate_environment", python_frame=r"(?<!_)validate_environment$"),
|
||||
mapping(rust_span="transform_request", python_frame=r"(?<!async_)transform_request$"),
|
||||
mapping(rust_span="execute_chat_completions_provider_call"),
|
||||
|
|
@ -110,7 +110,7 @@ SPEC: Final = RouteSpec(
|
|||
)
|
||||
BEDROCK_COMMON_MAPPINGS: Final = (
|
||||
mapping(rust_span="chat_completions_provider_config"),
|
||||
mapping(rust_span="supported_openai_params"),
|
||||
mapping(rust_span="supported_provider_params"),
|
||||
mapping(rust_span="execute_chat_completions_provider_call"),
|
||||
mapping(rust_span="validate_environment"),
|
||||
mapping(rust_span="http_request", python_frame=r"AsyncHTTPHandler\.post$|HTTPHandler\.post$"),
|
||||
|
|
|
|||
|
|
@ -12,10 +12,10 @@ COMMON_MAPPINGS: Final = (
|
|||
mapping(rust_span="prepare_ocr_call", python_frame=r"ocr/main\.py:\d+ _prepare_ocr_request$"),
|
||||
mapping(rust_span="ocr_provider_config", python_frame=r"ProviderConfigManager\.get_provider_ocr_config$"),
|
||||
mapping(rust_span="supported_ocr_params", python_frame=r"get_supported_ocr_params$"),
|
||||
mapping(rust_span="map_ocr_params", python_frame=r"(?<!async_)map_ocr_params$"),
|
||||
mapping(rust_span="map_params", python_frame=r"(?<!async_)map_ocr_params$"),
|
||||
mapping(rust_span="validate_environment", python_frame=r"(?<!_)validate_environment$"),
|
||||
mapping(rust_span="complete_url", python_frame=r"get_complete_url$"),
|
||||
mapping(rust_span="transform_ocr_request", python_frame=r"(?<!async_)transform_ocr_request$"),
|
||||
mapping(rust_span="transform_request", python_frame=r"(?<!async_)transform_ocr_request$"),
|
||||
mapping(rust_span="http_request", python_frame=r"AsyncHTTPHandler\.post$|HTTPHandler\.post$"),
|
||||
)
|
||||
|
||||
|
|
@ -38,7 +38,7 @@ SYNC_MAPPINGS: Final = (
|
|||
mapping(rust_span="execute_ocr_provider_call", python_frame=r"BaseLLMHTTPHandler\.ocr$"),
|
||||
mapping(span="python_transform_ocr_response_wrapper", python_frame=r"BaseLLMHTTPHandler\._transform_ocr_response$"),
|
||||
mapping(
|
||||
rust_span="transform_ocr_response",
|
||||
rust_span="transform_response",
|
||||
python_frame=r"MistralOCRConfig\.transform_ocr_response$",
|
||||
),
|
||||
)
|
||||
|
|
@ -48,7 +48,7 @@ ASYNC_MAPPINGS: Final = (
|
|||
mapping(span="python_ocr_wrapper", python_frame=r"BaseLLMHTTPHandler\.ocr$"),
|
||||
mapping(rust_span="execute_ocr_provider_call", python_frame=r"BaseLLMHTTPHandler\.async_ocr$"),
|
||||
mapping(
|
||||
rust_span="transform_ocr_response",
|
||||
rust_span="transform_response",
|
||||
python_frame=r"MistralOCRConfig\.transform_ocr_response$",
|
||||
),
|
||||
)
|
||||
|
|
@ -71,7 +71,7 @@ CALLBACK_FAILURE_ASYNC_MAPPINGS: Final = (
|
|||
AZURE_COMMON_MAPPINGS: Final = (
|
||||
*COMMON_MAPPINGS[:7],
|
||||
mapping(
|
||||
rust_span="transform_ocr_request",
|
||||
rust_span="transform_request",
|
||||
python_frame=(
|
||||
r"AzureAIOCRConfig\.(?:async_)?transform_ocr_request$"
|
||||
r"|MistralOCRConfig\.transform_ocr_request$"
|
||||
|
|
@ -87,7 +87,7 @@ AZURE_SYNC_MAPPINGS: Final = (
|
|||
python_frame=r"BaseLLMHTTPHandler\._transform_ocr_response$",
|
||||
),
|
||||
mapping(
|
||||
rust_span="transform_ocr_response",
|
||||
rust_span="transform_response",
|
||||
python_frame=r"MistralOCRConfig\.transform_ocr_response$",
|
||||
),
|
||||
)
|
||||
|
|
@ -96,7 +96,7 @@ AZURE_ASYNC_MAPPINGS: Final = (
|
|||
mapping(span="python_ocr_wrapper", python_frame=r"BaseLLMHTTPHandler\.ocr$"),
|
||||
mapping(rust_span="execute_ocr_provider_call", python_frame=r"BaseLLMHTTPHandler\.async_ocr$"),
|
||||
mapping(
|
||||
rust_span="transform_ocr_response",
|
||||
rust_span="transform_response",
|
||||
python_frame=r"MistralOCRConfig\.transform_ocr_response$",
|
||||
),
|
||||
)
|
||||
|
|
@ -252,7 +252,7 @@ def _azure_document_intelligence_fixture(engine: Engine, base_url: str) -> Route
|
|||
VERTEX_COMMON_MAPPINGS: Final = (
|
||||
*COMMON_MAPPINGS[:7],
|
||||
mapping(
|
||||
rust_span="transform_ocr_request",
|
||||
rust_span="transform_request",
|
||||
python_frame=(
|
||||
r"VertexAIOCRConfig\.(?:async_)?transform_ocr_request$"
|
||||
r"|MistralOCRConfig\.transform_ocr_request$"
|
||||
|
|
@ -264,13 +264,13 @@ VERTEX_SYNC_MAPPINGS: Final = (
|
|||
*VERTEX_COMMON_MAPPINGS,
|
||||
mapping(rust_span="execute_ocr_provider_call", python_frame=r"BaseLLMHTTPHandler\.ocr$"),
|
||||
mapping(span="python_transform_ocr_response_wrapper", python_frame=r"BaseLLMHTTPHandler\._transform_ocr_response$"),
|
||||
mapping(rust_span="transform_ocr_response", python_frame=r"MistralOCRConfig\.transform_ocr_response$"),
|
||||
mapping(rust_span="transform_response", python_frame=r"MistralOCRConfig\.transform_ocr_response$"),
|
||||
)
|
||||
VERTEX_ASYNC_MAPPINGS: Final = (
|
||||
*VERTEX_COMMON_MAPPINGS,
|
||||
mapping(span="python_ocr_wrapper", python_frame=r"BaseLLMHTTPHandler\.ocr$"),
|
||||
mapping(rust_span="execute_ocr_provider_call", python_frame=r"BaseLLMHTTPHandler\.async_ocr$"),
|
||||
mapping(rust_span="transform_ocr_response", python_frame=r"MistralOCRConfig\.transform_ocr_response$"),
|
||||
mapping(rust_span="transform_response", python_frame=r"MistralOCRConfig\.transform_ocr_response$"),
|
||||
)
|
||||
|
||||
DEEPSEEK_COMMON_MAPPINGS: Final = (
|
||||
|
|
@ -278,16 +278,16 @@ DEEPSEEK_COMMON_MAPPINGS: Final = (
|
|||
mapping(rust_span="prepare_ocr_call", python_frame=r"ocr/main\.py:\d+ _prepare_ocr_request$"),
|
||||
mapping(rust_span="ocr_provider_config", python_frame=r"ProviderConfigManager\.get_provider_ocr_config$"),
|
||||
mapping(rust_span="supported_ocr_params", python_frame=r"get_supported_ocr_params$"),
|
||||
mapping(rust_span="map_ocr_params", python_frame=r"(?<!async_)map_ocr_params$"),
|
||||
mapping(rust_span="map_params", python_frame=r"(?<!async_)map_ocr_params$"),
|
||||
mapping(rust_span="validate_environment", python_frame=r"(?<!_)validate_environment$"),
|
||||
mapping(rust_span="complete_url", python_frame=r"get_complete_url$"),
|
||||
mapping(
|
||||
rust_span="transform_ocr_request",
|
||||
rust_span="transform_request",
|
||||
python_frame=r"VertexAIDeepSeekOCRConfig\.transform_ocr_request$",
|
||||
),
|
||||
mapping(rust_span="http_request", python_frame=r"AsyncHTTPHandler\.post$|HTTPHandler\.post$"),
|
||||
mapping(
|
||||
rust_span="transform_ocr_response",
|
||||
rust_span="transform_response",
|
||||
python_frame=r"VertexAIDeepSeekOCRConfig\.transform_ocr_response$",
|
||||
),
|
||||
)
|
||||
|
|
@ -313,13 +313,13 @@ DOCUMENT_INTELLIGENCE_COMMON_MAPPINGS: Final = (
|
|||
mapping(
|
||||
rust_span="supported_ocr_params", python_frame=r"AzureDocumentIntelligenceOCRConfig\.get_supported_ocr_params$"
|
||||
),
|
||||
mapping(rust_span="map_ocr_params", python_frame=r"AzureDocumentIntelligenceOCRConfig\.map_ocr_params$"),
|
||||
mapping(rust_span="map_params", python_frame=r"AzureDocumentIntelligenceOCRConfig\.map_ocr_params$"),
|
||||
mapping(
|
||||
rust_span="validate_environment", python_frame=r"AzureDocumentIntelligenceOCRConfig\.validate_environment$"
|
||||
),
|
||||
mapping(rust_span="complete_url", python_frame=r"AzureDocumentIntelligenceOCRConfig\.get_complete_url$"),
|
||||
mapping(
|
||||
rust_span="transform_ocr_request", python_frame=r"AzureDocumentIntelligenceOCRConfig\.transform_ocr_request$"
|
||||
rust_span="transform_request", python_frame=r"AzureDocumentIntelligenceOCRConfig\.transform_ocr_request$"
|
||||
),
|
||||
mapping(rust_span="http_request", python_frame=r"AsyncHTTPHandler\.post$|HTTPHandler\.post$"),
|
||||
mapping(
|
||||
|
|
@ -327,7 +327,7 @@ DOCUMENT_INTELLIGENCE_COMMON_MAPPINGS: Final = (
|
|||
python_frame=r"AzureDocumentIntelligenceOCRConfig\._poll_operation_(?:sync|async)$",
|
||||
),
|
||||
mapping(
|
||||
rust_span="transform_ocr_response",
|
||||
rust_span="transform_response",
|
||||
python_frame=r"AzureDocumentIntelligenceOCRConfig\._transform_completed_response$",
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,19 +13,18 @@ from ...models import RouteFixture, RouteSpec, TraceScenario, TraceSuite
|
|||
MAPPINGS: Final = (
|
||||
mapping(rust_span="prepare_audio_transcription_provider_call"),
|
||||
mapping(span="get_non_default_params", python_frame=r"get_non_default_transcription_params$"),
|
||||
mapping(rust_span="map_transcription_params", python_frame=r"get_optional_params_transcription$"),
|
||||
mapping(rust_span="map_params", python_frame=r"get_optional_params_transcription$"),
|
||||
mapping(
|
||||
span="python_provider_config",
|
||||
python_frame=r"ProviderConfigManager\.get_provider_audio_transcription_config$",
|
||||
),
|
||||
mapping(rust_span="provider_config"),
|
||||
mapping(rust_span="supported_transcription_params"),
|
||||
mapping(rust_span="transform_transcription_request"),
|
||||
mapping(rust_span="transform_request"),
|
||||
mapping(
|
||||
rust_span="execute_audio_transcription_provider_call",
|
||||
python_frame=r"BedrockAudioTranscriptionRustDispatch\.(?:async_)?audio_transcriptions$",
|
||||
),
|
||||
mapping(rust_span="transform_transcription_response"),
|
||||
mapping(rust_span="transform_response"),
|
||||
mapping(rust_span="http_request"),
|
||||
)
|
||||
|
||||
|
|
@ -38,7 +37,7 @@ ASYNC_MAPPINGS: Final = (
|
|||
mapping(span="python_transcription_wrapper", python_frame=r"main\.py:\d+ transcription$"),
|
||||
*MAPPINGS[:2],
|
||||
mapping(span="python_map_transcription_params", python_frame=r"get_optional_params_transcription$"),
|
||||
mapping(rust_span="map_transcription_params"),
|
||||
mapping(rust_span="map_params"),
|
||||
*MAPPINGS[3:],
|
||||
)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue