scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-06 02:48:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 103

refactor(rust): add types and core-utils crates, unify the provider error

Browse source 
litellm-types mirrors litellm/types (chat, Anthropic Messages and Responses
websocket data) and litellm-core-utils mirrors litellm/litellm_core_utils
(provider resolution, prompt factory, core helpers, call arguments). Route
request types move up to their core route module, the transform contracts
move into base_llm, and the three duplicated provider error enums become one
Error in base_llm/chat/transformation.rs, mirroring BaseLLMException.

The empty-text placeholder goes back to the value Python's factory.py uses;
the provider extraction had changed it to a single space.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-09-17 22:01:14 -07:00
parent b1f9da79a8
commit 904c679595
101 changed files with 815 additions and 749 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
litellm-rust/Cargo.lock generated
View file

@ -2036,8 +2036,10 @@ dependencies = [
"litellm-auth-azure",
"litellm-auth-gcp",
"litellm-callbacks",
"litellm-core-utils",
"litellm-framing",
"litellm-providers",
"litellm-types",
"mime_guess",
"moka",
"rand 0.8.7",
@ -2061,6 +2063,20 @@ dependencies = [
"veil",
]
[[package]]
name = "litellm-core-utils"
version = "0.1.0"
dependencies = [
"litellm-types",
"rstest",
"serde",
"serde_json",
"serde_path_to_error",
"serde_with",
"thiserror 2.0.19",
"url",
]
[[package]]
name = "litellm-framing"
version = "0.1.0"
@ -2096,6 +2112,8 @@ version = "0.1.0"
dependencies = [
"litellm-auth",
"litellm-auth-aws",
"litellm-core-utils",
"litellm-types",
"rstest",
"serde",
"serde_json",
@ -2114,6 +2132,7 @@ dependencies = [
"litellm-core",
"litellm-host-python",
"litellm-token-counter",
"litellm-types",
"pyo3",
"pyo3-async-runtimes",
"rstest",
@ -2140,6 +2159,14 @@ dependencies = [
"unicode-normalization-alignments",
]
[[package]]
name = "litellm-types"
version = "0.1.0"
dependencies = [
"serde",
"serde_json",
]
[[package]]
name = "litemap"
version = "0.8.2"

2
litellm-rust/Cargo.toml
View file

@ -18,6 +18,8 @@ litellm-auth-aws = { path = "crates/auth-aws" }
litellm-auth-azure = { path = "crates/auth-azure" }
litellm-auth-gcp = { path = "crates/auth-gcp" }
litellm-providers = { path = "crates/providers" }
litellm-types = { path = "crates/types" }
litellm-core-utils = { path = "crates/core-utils" }
litellm-cache = { path = "crates/cache" }
litellm-cache-memory = { path = "crates/cache-memory" }
litellm-token-counter = { path = "crates/token-counter" }

18
litellm-rust/crates/core-utils/Cargo.toml Normal file
View file

@ -0,0 +1,18 @@
[package]
name = "litellm-core-utils"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
litellm-types.workspace = true
serde.workspace = true
serde_json.workspace = true
serde_path_to_error = "0.1"
serde_with.workspace = true
thiserror.workspace = true
url.workspace = true
[dev-dependencies]
rstest.workspace = true

2
litellm-rust/crates/core/src/call_arguments.rs → litellm-rust/crates/core-utils/src/call_arguments.rs
View file

@ -8,7 +8,7 @@ use serde_json::{Map, Value};
pub struct CallArguments(Map<String, Value>);
impl CallArguments {
pub(crate) fn select(&self, names: &[&str]) -> Map<String, Value> {
pub fn select(&self, names: &[&str]) -> Map<String, Value> {
self.iter()
.filter(|(name, _)| names.contains(&name.as_str()))
.map(|(name, value)| (name.clone(), value.clone()))

13
litellm-rust/crates/providers/src/chat/response_utils.rs → litellm-rust/crates/core-utils/src/core_helpers.rs
View file

@ -2,7 +2,7 @@
use std::time::{SystemTime, UNIX_EPOCH};
use super::types::{ChatCompletionsUsage, PromptTokensDetails};
use litellm_types::utils::{ChatCompletionsUsage, PromptTokensDetails};
/// OpenAI finish reasons, mirroring Python's `_FINISH_REASON_MAP` for the
/// reasons the providers on this route can emit. Python warns and falls back to
@ -54,6 +54,17 @@ pub fn unix_now() -> u64 {
.map_or(0, |elapsed| elapsed.as_secs())
}
pub fn json_type_name(value: &serde_json::Value) -> &'static str {
match value {
serde_json::Value::Null => "null",
serde_json::Value::Bool(_) => "boolean",
serde_json::Value::Number(_) => "number",
serde_json::Value::String(_) => "string",
serde_json::Value::Array(_) => "array",
serde_json::Value::Object(_) => "object",
}
}
#[cfg(test)]
mod tests {
use super::*;

34
litellm-rust/crates/core/src/litellm_core_utils/get_llm_provider_logic.rs → litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs
View file

@ -1,4 +1,36 @@
pub use litellm_providers::provider_resolution::{CustomLlmProvider, get_custom_llm_provider};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CustomLlmProvider<'a> {
pub model: &'a str,
pub custom_llm_provider: &'a str,
}
pub fn get_custom_llm_provider<'a>(
model: &'a str,
custom_llm_provider: Option<&'a str>,
) -> Option<CustomLlmProvider<'a>> {
if let Some(custom_llm_provider) = custom_llm_provider.filter(|provider| !provider.is_empty()) {
return Some(CustomLlmProvider {
model: strip_custom_llm_provider_prefix(model, custom_llm_provider),
custom_llm_provider,
});
}
let (custom_llm_provider, model) = model.split_once('/')?;
if custom_llm_provider.is_empty() || model.is_empty() {
return None;
}
Some(CustomLlmProvider {
model,
custom_llm_provider,
})
}
fn strip_custom_llm_provider_prefix<'a>(model: &'a str, custom_llm_provider: &str) -> &'a str {
model
.strip_prefix(custom_llm_provider)
.and_then(|model| model.strip_prefix('/'))
.unwrap_or(model)
}
#[cfg(test)]
mod tests {

7
litellm-rust/crates/core-utils/src/lib.rs Normal file
View file

@ -0,0 +1,7 @@
pub mod call_arguments;
pub mod core_helpers;
pub mod get_llm_provider_logic;
pub mod params;
pub mod prompt_templates;
pub mod serde_compat;
pub mod url_utils;

0
litellm-rust/crates/core/src/params.rs → litellm-rust/crates/core-utils/src/params.rs
View file

12
litellm-rust/crates/providers/src/chat/conversation.rs → litellm-rust/crates/core-utils/src/prompt_templates/factory.rs
View file

@ -10,8 +10,10 @@
//! `_bedrock_converse_messages_pt` for the text-only surface this route
//! accepts; anything richer is declined upstream by the capability gate.
use super::types::{ChatMessage, ChatMessageContent};
use crate::chat::EMPTY_TEXT_PLACEHOLDER;
use litellm_types::llms::openai::{ChatMessage, ChatMessageContent};
pub const EMPTY_TEXT_PLACEHOLDER: &str =
"[System: Empty message content sanitised to satisfy protocol]";
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum TurnRole {
@ -203,8 +205,10 @@ mod tests {
{"role": "assistant", "content": " "},
{"role": "user", "content": "real"}
])));
assert_eq!(conversation.turns[0].texts, vec![EMPTY_TEXT_PLACEHOLDER]);
assert_eq!(conversation.turns[1].texts, vec![EMPTY_TEXT_PLACEHOLDER]);
// Must equal `_EMPTY_TEXT_PLACEHOLDER` in litellm/litellm_core_utils/prompt_templates/factory.py
let placeholder = "[System: Empty message content sanitised to satisfy protocol]";
assert_eq!(conversation.turns[0].texts, vec![placeholder]);
assert_eq!(conversation.turns[1].texts, vec![placeholder]);
}
#[test]

1
litellm-rust/crates/core-utils/src/prompt_templates/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod factory;

4
litellm-rust/crates/core/src/serde_compat.rs → litellm-rust/crates/core-utils/src/serde_compat.rs
View file

@ -2,8 +2,8 @@ use serde::{Deserialize, Deserializer, de::Error};
use serde_json::Value;
use serde_with::DeserializeAs;
pub(crate) struct LaxI64;
pub(crate) struct FiniteF64;
pub struct LaxI64;
pub struct FiniteF64;
impl<'de> DeserializeAs<'de, i64> for LaxI64 {
fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<i64, D::Error> {

19
litellm-rust/crates/core/src/url_utils.rs → litellm-rust/crates/core-utils/src/url_utils.rs
View file

@ -3,33 +3,30 @@ use std::marker::PhantomData;
use url::Url;
#[derive(Debug, thiserror::Error)]
pub(crate) enum ApiUrlError {
pub enum ApiUrlError {
#[error("invalid URL: {0}")]
Parse(#[from] url::ParseError),
#[error("URL cannot be used as a base")]
CannotBeBase,
}
pub(crate) struct Base;
pub(crate) struct Complete;
pub struct Base;
pub struct Complete;
pub(crate) struct ApiUrl<State> {
pub struct ApiUrl<State> {
url: Url,
state: PhantomData<State>,
}
impl ApiUrl<Base> {
pub(crate) fn parse(value: &str) -> Result<Self, ApiUrlError> {
pub fn parse(value: &str) -> Result<Self, ApiUrlError> {
Ok(Self {
url: Url::parse(value.trim())?,
state: PhantomData,
})
}
pub(crate) fn complete_path(
mut self,
target: &[&str],
) -> Result<ApiUrl<Complete>, ApiUrlError> {
pub fn complete_path(mut self, target: &[&str]) -> Result<ApiUrl<Complete>, ApiUrlError> {
let existing: Vec<String> = self
.url
.path_segments()
@ -59,7 +56,7 @@ impl ApiUrl<Base> {
}
impl ApiUrl<Complete> {
pub(crate) fn append_query_pairs<'a>(
pub fn append_query_pairs<'a>(
mut self,
pairs: impl IntoIterator<Item = (&'a str, &'a str)>,
) -> Self {
@ -67,7 +64,7 @@ impl ApiUrl<Complete> {
self
}
pub(crate) fn into_string(self) -> String {
pub fn into_string(self) -> String {
self.url.into()
}
}

2
litellm-rust/crates/core/Cargo.toml
View file

@ -7,6 +7,8 @@ repository.workspace = true
autotests = false
[dependencies]
litellm-types.workspace = true
litellm-core-utils.workspace = true
litellm-callbacks.workspace = true
bytes.workspace = true
futures-util.workspace = true

25
litellm-rust/crates/core/src/audio_transcription/error.rs
View file

@ -1,3 +1,5 @@
use litellm_providers::base_llm::chat::transformation::Error as LlmError;
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum Error {
#[error("expected {expected}, got {actual}")]
@ -25,22 +27,15 @@ pub enum Error {
Aws(#[from] litellm_auth_aws::Error),
}
impl From<litellm_providers::audio_transcription::Error> for Error {
fn from(error: litellm_providers::audio_transcription::Error) -> Self {
impl From<LlmError> for Error {
fn from(error: LlmError) -> Self {
match error {
litellm_providers::audio_transcription::Error::InvalidType { expected, actual } => {
Self::InvalidType { expected, actual }
}
litellm_providers::audio_transcription::Error::MissingField(field) => {
Self::MissingField(field)
}
litellm_providers::audio_transcription::Error::InvalidRequest(message) => {
Self::InvalidRequest(message)
}
litellm_providers::audio_transcription::Error::InvalidResponse(message) => {
Self::InvalidResponse(message)
}
litellm_providers::audio_transcription::Error::Auth(error) => Self::Auth(error),
LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual },
LlmError::MissingField(field) => Self::MissingField(field),
LlmError::InvalidRequest(message) => Self::InvalidRequest(message),
LlmError::InvalidResponse(message) => Self::InvalidResponse(message),
LlmError::Unsupported(reason) => Self::Unsupported(reason),
LlmError::Auth(error) => Self::Auth(error),
}
}
}

7
litellm-rust/crates/core/src/audio_transcription/handler.rs
View file

@ -1,7 +1,10 @@
use serde_json::Value;
use super::{Error, client::http_client, types::ProviderAudioTranscriptionRequest};
use crate::http_utils::{http_request, truncate_error_body};
use super::{Error, client::http_client};
use crate::{
audio_transcription::types::ProviderAudioTranscriptionRequest,
http_utils::{http_request, truncate_error_body},
};
pub async fn execute_audio_transcription_provider_call(
request: ProviderAudioTranscriptionRequest,

5
litellm-rust/crates/core/src/audio_transcription/mod.rs
View file

@ -1,13 +1,14 @@
mod error;
pub mod types;
pub use error::Error;
mod client;
mod handler;
mod prepare;
pub use handler::execute_audio_transcription_provider_call;
pub use litellm_providers::audio_transcription::types;
pub use prepare::prepare_audio_transcription_provider_call;
use serde_json::Value;
pub use types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest};
use crate::audio_transcription::types::AudioTranscriptionRequest;
pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
execute_audio_transcription_provider_call(prepare_audio_transcription_provider_call(request)?)

8
litellm-rust/crates/core/src/audio_transcription/prepare.rs
View file

@ -1,3 +1,4 @@
use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
use litellm_providers::{
base_llm::audio_transcription::transformation::{
AudioTranscriptionAuth, BaseAudioTranscriptionConfig,
@ -5,13 +6,10 @@ use litellm_providers::{
bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
};
use super::{
Error,
types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest},
};
use super::Error;
use crate::{
audio_transcription::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest},
http_utils::{has_header, string_headers},
litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider},
};
fn provider_config(provider: &str) -> Option<&'static dyn BaseAudioTranscriptionConfig> {

3
litellm-rust/crates/core/src/audio_transcription/tests.rs
View file

@ -6,7 +6,8 @@ use std::{
use serde_json::{Map, json};
use super::{audio_transcription, types::AudioTranscriptionRequest};
use super::audio_transcription;
use crate::audio_transcription::types::AudioTranscriptionRequest;
#[tokio::test]
async fn bedrock_request_is_signed_and_contains_audio() {

24
litellm-rust/crates/providers/src/audio_transcription/types.rs → litellm-rust/crates/core/src/audio_transcription/types.rs
View file

@ -1,11 +1,9 @@
use std::time::Duration;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use crate::base_llm::audio_transcription::transformation::{
use litellm_providers::base_llm::audio_transcription::transformation::{
AudioTranscriptionAuth, BaseAudioTranscriptionConfig,
};
use serde_json::{Map, Value};
pub struct AudioTranscriptionRequest<'a> {
pub model: &'a str,
@ -52,21 +50,3 @@ impl ProviderAudioTranscriptionRequest {
Self { body, ..self }
}
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AudioTranscriptionRequestData {
pub body: Value,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AudioTranscriptionResponseData {
pub text: String,
}
impl AudioTranscriptionResponseData {
pub fn into_json(self) -> Value {
serde_json::json!({
"text": self.text,
})
}
}

21
litellm-rust/crates/core/src/chat_completions/error.rs
View file

@ -1,3 +1,5 @@
use litellm_providers::base_llm::chat::transformation::Error as LlmError;
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum Error {
#[error("expected {expected}, got {actual}")]
@ -25,18 +27,15 @@ pub enum Error {
Aws(#[from] litellm_auth_aws::Error),
}
impl From<litellm_providers::chat::Error> for Error {
fn from(error: litellm_providers::chat::Error) -> Self {
impl From<LlmError> for Error {
fn from(error: LlmError) -> Self {
match error {
litellm_providers::chat::Error::MissingField(field) => Self::MissingField(field),
litellm_providers::chat::Error::InvalidRequest(message) => {
Self::InvalidRequest(message)
}
litellm_providers::chat::Error::InvalidResponse(message) => {
Self::InvalidResponse(message)
}
litellm_providers::chat::Error::Unsupported(reason) => Self::Unsupported(reason),
litellm_providers::chat::Error::Auth(error) => Self::Auth(error),
LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual },
LlmError::MissingField(field) => Self::MissingField(field),
LlmError::InvalidRequest(message) => Self::InvalidRequest(message),
LlmError::InvalidResponse(message) => Self::InvalidResponse(message),
LlmError::Unsupported(reason) => Self::Unsupported(reason),
LlmError::Auth(error) => Self::Auth(error),
}
}
}

18
litellm-rust/crates/core/src/chat_completions/handler.rs
View file

@ -1,16 +1,14 @@
use litellm_providers::base_llm::chat::transformation::ChatCompletionsAuth;
use litellm_providers::base_llm::chat::transformation::{
ChatCompletionsAuth, ProviderChatResponseData,
};
use litellm_types::utils::ChatCompletionsResponse;
use serde_json::Value;
use super::{
Error,
client::http_client,
prepare::prepare_provider_request,
types::{
ChatCompletionsResponse, ProviderChatCompletionsRequest, ProviderChatResponseData,
ResolvedChatCompletionsRequest,
},
use super::{Error, client::http_client, prepare::prepare_provider_request};
use crate::{
chat_completions::types::{ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest},
http_utils::{http_request, truncate_error_body},
};
use crate::http_utils::{http_request, truncate_error_body};
pub(super) async fn execute_chat_completions_provider_call(
request: ResolvedChatCompletionsRequest<'_>,

7
litellm-rust/crates/core/src/chat_completions/mod.rs
View file

@ -7,18 +7,19 @@
//! calls the provider, and returns a typed OpenAI-shaped response.
mod error;
pub mod types;
pub use error::Error;
mod client;
mod common_utils;
pub use litellm_providers::chat::{conversation, response_utils};
pub(crate) mod handler;
mod prepare;
pub mod streaming;
use handler::execute_chat_completions_provider_call;
pub use litellm_providers::chat::types;
use litellm_types::utils::ChatCompletionsResponse;
use prepare::{parse_messages, resolve_provider_config, resolve_request};
use serde_json::{Map, Value};
use types::{ChatCompletionsRequest, ChatCompletionsResponse};
use crate::chat_completions::types::ChatCompletionsRequest;
pub async fn chat_completions(
request: ChatCompletionsRequest<'_>,

10
litellm-rust/crates/core/src/chat_completions/prepare.rs
View file

@ -1,17 +1,17 @@
use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
use litellm_providers::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth};
use litellm_types::llms::openai::ChatMessage;
use serde_json::Value;
use super::{
Error,
common_utils::{chat_completions_provider_config, string_headers},
types::{
ChatCompletionsRequest, ChatMessage, ProviderChatCompletionsRequest,
ResolvedChatCompletionsRequest,
},
};
use crate::{
chat_completions::types::{
ChatCompletionsRequest, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest,
},
http_utils::has_header,
litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider},
};
pub(super) fn resolve_provider_config<'a>(

2
litellm-rust/crates/core/src/chat_completions/tests.rs
View file

@ -4,8 +4,8 @@ use serde_json::{Map, Value, json};
use super::{
Error,
prepare::{prepare_provider_request, resolve_request},
types::{ChatCompletionsRequest, ProviderChatCompletionsRequest},
};
use crate::chat_completions::types::{ChatCompletionsRequest, ProviderChatCompletionsRequest};
fn prepare_chat_completions_call(
request: ChatCompletionsRequest<'_>,

44
litellm-rust/crates/core/src/chat_completions/types.rs Normal file
View file

@ -0,0 +1,44 @@
use std::time::Duration;
use litellm_providers::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth};
use litellm_types::llms::openai::ChatMessage;
use serde_json::{Map, Value};
/// A `/chat/completions` call as it crosses into the core.
///
/// `optional_params` arrives already mapped to the provider's own parameter
/// names by the host, exactly as the messages route receives an already
/// Anthropic-shaped body. The core owns the conversation translation, the
/// provider call, and the response normalization.
pub struct ChatCompletionsRequest<'a> {
pub model: &'a str,
pub messages: Value,
pub optional_params: Map<String, Value>,
pub api_key: Option<&'a str>,
pub api_base: Option<&'a str>,
pub custom_llm_provider: Option<&'a str>,
pub extra_headers: Option<Map<String, Value>>,
pub timeout: Option<Duration>,
}
pub struct ResolvedChatCompletionsRequest<'a> {
pub model: String,
pub config: &'static dyn BaseConfig,
pub messages: Vec<ChatMessage>,
pub optional_params: Map<String, Value>,
pub api_key: Option<&'a str>,
pub api_base: Option<&'a str>,
pub extra_headers: Option<Map<String, Value>>,
pub timeout: Option<Duration>,
}
pub struct ProviderChatCompletionsRequest {
pub model: String,
pub config: &'static dyn BaseConfig,
pub url: String,
pub body: Value,
pub upstream_headers: Vec<(String, String)>,
pub auth: ChatCompletionsAuth,
pub optional_params: Map<String, Value>,
pub timeout: Option<Duration>,
}

12
litellm-rust/crates/core/src/http_utils.rs
View file

@ -6,6 +6,7 @@ pub struct HeaderError {
pub actual: &'static str,
}
use litellm_core_utils::core_helpers::json_type_name;
use serde_json::{Map, Value};
use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS;
@ -118,17 +119,6 @@ where
<Option<T> as serde::Deserialize>::deserialize(deserializer).map(Some)
}
pub fn json_type_name(value: &serde_json::Value) -> &'static str {
match value {
serde_json::Value::Null => "null",
serde_json::Value::Bool(_) => "bool",
serde_json::Value::Number(_) => "number",
serde_json::Value::String(_) => "string",
serde_json::Value::Array(_) => "array",
serde_json::Value::Object(_) => "object",
}
}
#[cfg(test)]
mod tests {
use serde_json::json;

5
litellm-rust/crates/core/src/lib.rs
View file

@ -1,19 +1,14 @@
pub mod audio_transcription;
pub mod call_arguments;
pub mod chat_completions;
pub mod constants;
pub mod error;
pub mod http_utils;
pub mod litellm_core_utils;
pub mod llms;
pub mod machine;
mod media;
pub mod messages;
pub mod ocr;
pub mod params;
pub mod responses;
mod serde_compat;
pub mod transport;
mod url_utils;
pub use error::Error;

1
litellm-rust/crates/core/src/litellm_core_utils/mod.rs
View file

@ -1 +0,0 @@
pub mod get_llm_provider_logic;

13
litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs
View file

@ -1,19 +1,16 @@
use std::collections::HashMap;
use litellm_types::{
llms::openai::{ChatCompletionThinkingBlock, ChatCompletionToolCallChunk},
utils::{ChatCompletionChunk, ChatCompletionsUsage},
};
use serde_json::Value;
use super::super::experimental_pass_through::messages::streaming::{
AnthropicContentBlock, AnthropicContentBlockDelta, AnthropicMessagesStreamEvent,
AnthropicStreamUsage,
};
use crate::chat_completions::{
Error,
streaming::StreamTransformer,
types::{
ChatCompletionChunk, ChatCompletionThinkingBlock, ChatCompletionToolCallChunk,
ChatCompletionsUsage,
},
};
use crate::chat_completions::{Error, streaming::StreamTransformer};
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum AnthropicJsonChunkType {

3
litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs
View file

@ -1,10 +1,11 @@
use litellm_providers::anthropic::experimental_pass_through::messages::transformation::resolve_anthropic_api_base;
use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse;
use serde::{Deserialize, Serialize};
use serde_json::Value;
use time::OffsetDateTime;
use url::Url;
use crate::messages::{Error, types::AnthropicMessagesResponse};
use crate::messages::Error;
const BATCHES_PATH_SUFFIX: &str = "/v1/messages/batches";

11
litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs
View file

@ -1,13 +1,8 @@
use litellm_types::llms::anthropic_messages::anthropic_request::{AnthropicMessage, SystemPrompt};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{
constants::ANTHROPIC_OAUTH_TOKEN_PREFIX,
messages::{
Error,
types::{AnthropicMessage, SystemPrompt},
},
};
use crate::{constants::ANTHROPIC_OAUTH_TOKEN_PREFIX, messages::Error};
const COUNT_TOKENS_ENDPOINT: &str = "https://api.anthropic.com/v1/messages/count_tokens";
const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01";
@ -97,10 +92,10 @@ impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation {
#[cfg(test)]
mod tests {
use litellm_types::llms::anthropic_messages::anthropic_request::MessageContent;
use serde_json::{Map, json};
use super::*;
use crate::messages::types::MessageContent;
fn message() -> AnthropicMessage {
AnthropicMessage {

3
litellm-rust/crates/core/src/llms/azure_ai/ocr/cohere_parse_transformation.rs
View file

@ -1,7 +1,7 @@
use litellm_core_utils::{call_arguments::CallArguments, url_utils::ApiUrl};
use serde_json::Value;
use crate::{
call_arguments::CallArguments,
llms::{
base_llm::ocr::transformation::{BaseOcrConfig, OcrRequestContext},
cohere::ocr::{
@ -15,7 +15,6 @@ use crate::{
document::{inline_remote_document, validate_inline_document},
types::{LiteLLMOcrResponse, OcrDocument, PreparedOcrRequest},
},
url_utils::ApiUrl,
};
#[derive(Default)]

8
litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs
View file

@ -3,6 +3,11 @@ use std::{collections::BTreeSet, time::Duration};
use base64::{Engine, engine::general_purpose::STANDARD};
use litellm_auth::{InputSource, Sourced};
use litellm_auth_azure::AzureAuthInputs;
use litellm_core_utils::{
call_arguments::CallArguments,
serde_compat::{FiniteF64, LaxI64},
url_utils::ApiUrl,
};
use reqwest::Url;
use serde::{Deserialize, Deserializer, Serialize};
use serde_json::{Map, Value};
@ -10,7 +15,6 @@ use serde_with::serde_as;
use tokio::time::Instant;
use crate::{
call_arguments::CallArguments,
constants::{
AZURE_DI_API_VERSION, AZURE_DI_DEFAULT_DPI, AZURE_DI_DEFAULT_HEIGHT,
AZURE_DI_DEFAULT_WIDTH, AZURE_DI_SUBSCRIPTION_HEADER, OCR_POLL_RETRY_SECS,
@ -31,8 +35,6 @@ use crate::{
ResolvedOcrCredentials,
},
},
serde_compat::{FiniteF64, LaxI64},
url_utils::ApiUrl,
};
const AZURE_DI_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY";

40
litellm-rust/crates/core/src/llms/azure_ai/ocr/transformation.rs
View file

@ -1,17 +1,21 @@
use litellm_auth::{InputSource, Sourced};
use litellm_auth_azure::AzureAuthInputs;
use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl};
use serde_json::Value;
use crate::call_arguments::CallArguments;
use crate::constants::AZURE_AI_OCR_PATH;
use crate::llms::base_llm::ocr::transformation::{BaseOcrConfig, OcrRequestContext};
use crate::llms::mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest};
use crate::ocr::OcrClient;
use crate::ocr::document::{inline_remote_document, validate_inline_document};
use crate::ocr::prepare::credential_env;
use crate::ocr::types::{LiteLLMOcrResponse, OcrConnection, OcrDocument, PreparedOcrRequest};
use crate::params::OpaqueParams;
use crate::url_utils::ApiUrl;
use crate::{
constants::AZURE_AI_OCR_PATH,
llms::{
base_llm::ocr::transformation::{BaseOcrConfig, OcrRequestContext},
mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest},
},
ocr::{
OcrClient,
document::{inline_remote_document, validate_inline_document},
prepare::credential_env,
types::{LiteLLMOcrResponse, OcrConnection, OcrDocument, PreparedOcrRequest},
},
};
const AZURE_AI_API_KEY_ENV: &str = "AZURE_AI_API_KEY";
const AZURE_AI_API_BASE_ENV: &str = "AZURE_AI_API_BASE";
@ -306,9 +310,9 @@ mod tests {
use serde_json::json;
use crate::ocr::LocalOcrHost;
use crate::ocr::test_support::{
MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request,
use crate::ocr::{
LocalOcrHost,
test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request},
};
#[tokio::test]
@ -388,16 +392,16 @@ mod tests {
assert!(error.to_string().contains("data URI"));
}
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use litellm_auth::{
ResolvedCredential, SecretValue, TokenFuture, TokenProvider, TokenProviderHandle,
};
use crate::ocr::LiteLLMOcrRequest;
use crate::ocr::test_support::header;
use crate::ocr::wire::decode_request;
use crate::ocr::{LiteLLMOcrRequest, test_support::header, wire::decode_request};
#[derive(Debug)]
struct CountingToken {

16
litellm-rust/crates/core/src/llms/base_llm/ocr/transformation.rs
View file

@ -1,17 +1,15 @@
use std::future::Future;
use litellm_core_utils::call_arguments::CallArguments;
use serde::{Serialize, de::DeserializeOwned};
use serde_json::Value;
use crate::{
call_arguments::CallArguments,
ocr::{
OcrClient,
route::OcrHost,
types::{
LiteLLMOcrResponse, OcrConnection, OcrCredentialInputs, OcrDocument, OcrResponseFormat,
PreparedOcrRequest, ResolvedOcrCredentials,
},
use crate::ocr::{
OcrClient,
route::OcrHost,
types::{
LiteLLMOcrResponse, OcrConnection, OcrCredentialInputs, OcrDocument, OcrResponseFormat,
PreparedOcrRequest, ResolvedOcrCredentials,
},
};

8
litellm-rust/crates/core/src/llms/cohere/ocr/transformation.rs
View file

@ -1,9 +1,13 @@
use litellm_core_utils::{
call_arguments::{CallArguments, parse_options},
serde_compat::LaxI64,
url_utils::ApiUrl,
};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use serde_with::serde_as;
use crate::{
call_arguments::{CallArguments, parse_options},
constants::{COHERE_API_KEY_ENV, COHERE_PARSE_API_BASE},
llms::base_llm::ocr::transformation::{BaseOcrConfig, decode_and_normalize_response},
ocr::{
@ -15,8 +19,6 @@ use crate::{
OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
},
},
serde_compat::LaxI64,
url_utils::ApiUrl,
};
const COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4//8/AAX+Av4N70a4AAAAAElFTkSuQmCC";

4
litellm-rust/crates/core/src/llms/mistral/ocr/transformation.rs
View file

@ -1,8 +1,8 @@
use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use crate::{
call_arguments::CallArguments,
constants::MISTRAL_OCR_API_BASE,
llms::base_llm::ocr::transformation::{BaseOcrConfig, decode_and_normalize_response},
ocr::{
@ -13,8 +13,6 @@ use crate::{
OcrUsageInfo, PreparedOcrRequest,
},
},
params::OpaqueParams,
url_utils::ApiUrl,
};
const MISTRAL_OCR_API_KEY_ENV_VAR: &str = "MISTRAL_API_KEY";

3
litellm-rust/crates/core/src/llms/openai/responses/transformation.rs
View file

@ -1,6 +1,7 @@
use litellm_types::responses::streaming_websocket::{ResponsesWsEvent, ResponsesWsTransformResult};
use crate::responses::{
Error,
types::{ResponsesWsEvent, ResponsesWsTransformResult},
websocket::{ResponsesWebSocketProviderConfig, enforce_model},
};

17
litellm-rust/crates/core/src/llms/reducto/ocr/transformation.rs
View file

@ -1,10 +1,14 @@
use std::collections::BTreeMap;
use litellm_core_utils::{
call_arguments::{CallArguments, compose_body},
params::OpaqueParams,
url_utils::ApiUrl,
};
use serde::{Deserialize, Deserializer, Serialize};
use serde_json::{Map, Value, json};
use crate::{
call_arguments::{CallArguments, compose_body},
constants::{REDUCTO_API_BASE, REDUCTO_API_KEY_ENV, REDUCTO_ID_PREFIX},
llms::base_llm::ocr::transformation::{
BaseOcrConfig, OcrRequestContext, decode_and_normalize_response,
@ -18,8 +22,6 @@ use crate::{
OcrUsageInfo, PreparedOcrRequest,
},
},
params::OpaqueParams,
url_utils::ApiUrl,
};
#[derive(Clone, Debug, Serialize, Deserialize)]
@ -70,9 +72,9 @@ struct ReductoResult {
#[serde_with::serde_as]
#[derive(Clone, Debug, Default, Deserialize)]
struct ReductoUsage {
#[serde_as(deserialize_as = "Option<crate::serde_compat::LaxI64>")]
#[serde_as(deserialize_as = "Option<litellm_core_utils::serde_compat::LaxI64>")]
pub num_pages: Option<i64>,
#[serde_as(deserialize_as = "Option<crate::serde_compat::FiniteF64>")]
#[serde_as(deserialize_as = "Option<litellm_core_utils::serde_compat::FiniteF64>")]
pub credits: Option<f64>,
}
@ -624,7 +626,10 @@ mod tests {
})
);
let absent = ReductoParseV3Config
.map_ocr_params(&crate::call_arguments::CallArguments::default(), "parse-v3")
.map_ocr_params(
&litellm_core_utils::call_arguments::CallArguments::default(),
"parse-v3",
)
.unwrap();
assert_eq!(serde_json::to_value(absent).unwrap(), json!({}));
}

14
litellm-rust/crates/core/src/llms/vertex_ai/ocr/deepseek_transformation.rs
View file

@ -1,10 +1,10 @@
use litellm_auth_gcp::{self as vertex, VertexConfig};
use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use super::transformation::VertexAiOcrConfig;
use crate::{
call_arguments::CallArguments,
llms::base_llm::ocr::transformation::{BaseOcrConfig, OcrRequestContext},
ocr::{
OcrClient,
@ -14,8 +14,6 @@ use crate::{
OcrUsageInfo, PreparedOcrRequest,
},
},
params::OpaqueParams,
url_utils::ApiUrl,
};
const DEFAULT_API_BASE: &str = "https://aiplatform.googleapis.com";
@ -82,7 +80,7 @@ enum DeepSeekContent {
#[derive(Deserialize)]
struct DeepSeekPage {
#[serde(default)]
#[serde_as(deserialize_as = "crate::serde_compat::LaxI64")]
#[serde_as(deserialize_as = "litellm_core_utils::serde_compat::LaxI64")]
index: i64,
#[serde(default)]
markdown: String,
@ -434,8 +432,12 @@ mod tests {
json!({})
);
assert_eq!(
crate::call_arguments::compose_body(&arguments, &json!({"model":"deepseek-ocr"}), &[])
.unwrap(),
litellm_core_utils::call_arguments::compose_body(
&arguments,
&json!({"model":"deepseek-ocr"}),
&[]
)
.unwrap(),
json!({"model":"deepseek-ocr","temperature":0.5,"extension":null})
);
}

4
litellm-rust/crates/core/src/llms/vertex_ai/ocr/transformation.rs
View file

@ -1,9 +1,9 @@
use litellm_auth_gcp::{self as vertex, VertexConfig};
use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl};
use serde_json::Value;
use super::common_utils::validate_destination;
use crate::{
call_arguments::CallArguments,
llms::{
base_llm::ocr::transformation::{BaseOcrConfig, OcrEnvironment, OcrRequestContext},
mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest},
@ -14,8 +14,6 @@ use crate::{
prepare::credential_env,
types::{LiteLLMOcrResponse, OcrConnection, OcrDocument, PreparedOcrRequest},
},
params::OpaqueParams,
url_utils::ApiUrl,
};
const DEFAULT_LOCATION: &str = "us-central1";

21
litellm-rust/crates/core/src/messages/error.rs
View file

@ -1,3 +1,5 @@
use litellm_providers::base_llm::chat::transformation::Error as LlmError;
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum Error {
#[error("invalid provider: {0}")]
@ -28,18 +30,15 @@ pub enum Error {
InvalidBedrockBase64(String),
}
impl From<litellm_providers::messages::Error> for Error {
fn from(error: litellm_providers::messages::Error) -> Self {
impl From<LlmError> for Error {
fn from(error: LlmError) -> Self {
match error {
litellm_providers::messages::Error::MissingField(field) => Self::MissingField(field),
litellm_providers::messages::Error::InvalidRequest(message) => {
Self::InvalidRequest(message)
}
litellm_providers::messages::Error::InvalidResponse(message) => {
Self::InvalidResponse(message)
}
litellm_providers::messages::Error::Unsupported(reason) => Self::Unsupported(reason),
litellm_providers::messages::Error::Auth(error) => Self::Auth(error),
error @ LlmError::InvalidType { .. } => Self::InvalidRequest(error.to_string()),
LlmError::MissingField(field) => Self::MissingField(field),
LlmError::InvalidRequest(message) => Self::InvalidRequest(message),
LlmError::InvalidResponse(message) => Self::InvalidResponse(message),
LlmError::Unsupported(reason) => Self::Unsupported(reason),
LlmError::Auth(error) => Self::Auth(error),
}
}
}

12
litellm-rust/crates/core/src/messages/handler.rs
View file

@ -1,11 +1,13 @@
use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse;
use super::{
Error,
client::http_client,
common_utils::truncate_error_body,
Error, client::http_client, common_utils::truncate_error_body,
prepare::prepare_provider_request,
types::{AnthropicMessagesResponse, MessagesRequest},
};
use crate::{constants::ANTHROPIC_MESSAGES_PROVIDER, http_utils::http_request};
use crate::{
constants::ANTHROPIC_MESSAGES_PROVIDER, http_utils::http_request,
messages::types::MessagesRequest,
};
pub(super) async fn execute_messages_provider_call(
request: MessagesRequest<'_>,

6
litellm-rust/crates/core/src/messages/mod.rs
View file

@ -8,14 +8,16 @@
//! can splice the event stream to its own caller.
mod error;
pub mod types;
pub use error::Error;
mod client;
mod common_utils;
mod handler;
mod prepare;
use handler::{execute_messages_provider_call, execute_messages_provider_stream};
pub use litellm_providers::messages::types;
use types::{AnthropicMessagesResponse, MessagesRequest};
use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse;
use crate::messages::types::MessagesRequest;
pub async fn messages(request: MessagesRequest<'_>) -> Result<AnthropicMessagesResponse, Error> {
execute_messages_provider_call(request).await

6
litellm-rust/crates/core/src/messages/prepare.rs
View file

@ -1,3 +1,4 @@
use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
use litellm_providers::base_llm::anthropic_messages::transformation::{
BaseAnthropicMessagesConfig, MessagesAuthStrategy,
};
@ -6,11 +7,8 @@ use serde_json::{Map, Value};
use super::{
Error,
common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers},
types::{MessagesRequest, ProviderMessagesRequest},
};
use crate::litellm_core_utils::get_llm_provider_logic::{
CustomLlmProvider, get_custom_llm_provider,
};
use crate::messages::types::{MessagesRequest, ProviderMessagesRequest};
pub(super) fn prepare_provider_request(
request: MessagesRequest<'_>,

2
litellm-rust/crates/core/src/messages/tests.rs
View file

@ -12,8 +12,8 @@ use super::{
has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body,
},
messages,
types::MessagesRequest,
};
use crate::messages::types::MessagesRequest;
async fn read_http_request(socket: &mut TcpStream) -> String {
let mut request = Vec::new();

24
litellm-rust/crates/core/src/messages/types.rs Normal file
View file

@ -0,0 +1,24 @@
use std::time::Duration;
use litellm_providers::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig;
use serde_json::{Map, Value};
pub struct MessagesRequest<'a> {
pub model: &'a str,
pub body: Value,
pub api_key: Option<&'a str>,
pub api_base: Option<&'a str>,
pub custom_llm_provider: Option<&'a str>,
pub extra_headers: Option<Map<String, Value>>,
pub timeout: Option<Duration>,
}
pub struct ProviderMessagesRequest {
pub provider: String,
pub model: String,
pub config: &'static dyn BaseAnthropicMessagesConfig,
pub url: String,
pub body: Value,
pub upstream_headers: Vec<(String, String)>,
pub timeout: Option<Duration>,
}

3
litellm-rust/crates/core/src/ocr/arguments.rs
View file

@ -1,5 +1,6 @@
use litellm_core_utils::call_arguments::ArgumentSpec;
use super::provider_config::{OcrConfigKind, resolve_provider_config};
use crate::call_arguments::ArgumentSpec;
const COMMON_OPTION_FIELDS: &[&str] = &["req_format", "extra_body", "max_response_bytes"];
const AZURE_AUTH_OPTION_FIELDS: &[&str] = &[

6
litellm-rust/crates/core/src/ocr/error.rs
View file

@ -97,13 +97,13 @@ pub enum Error {
#[error(transparent)]
Transport(#[from] crate::transport::Error),
#[error(transparent)]
Params(#[from] crate::params::Error),
Params(#[from] litellm_core_utils::params::Error),
#[error(transparent)]
Headers(#[from] crate::http_utils::HeaderError),
}
impl From<crate::call_arguments::ArgumentError> for Error {
fn from(error: crate::call_arguments::ArgumentError) -> Self {
impl From<litellm_core_utils::call_arguments::ArgumentError> for Error {
fn from(error: litellm_core_utils::call_arguments::ArgumentError) -> Self {
Self::RequestField {
path: format!("optional_params.{}", error.path),
}

13
litellm-rust/crates/core/src/ocr/prepare.rs
View file

@ -2,9 +2,11 @@ use litellm_callbacks::event::{Passthrough, RequestContext, WireRequest};
use serde::Serialize;
use serde_json::{Map, Value};
use super::OcrClient;
use super::route::OcrHost;
use super::types::{OcrConnection, OcrDocument, PreparedOcrRequest, ResolvedOcrRequest};
use super::{
OcrClient,
route::OcrHost,
types::{OcrConnection, OcrDocument, PreparedOcrRequest, ResolvedOcrRequest},
};
pub(crate) async fn transform_request_body<B>(
client: &OcrClient,
@ -17,7 +19,7 @@ pub(crate) async fn transform_request_body<B>(
where
B: Serialize,
{
let composed = crate::call_arguments::compose_body(
let composed = litellm_core_utils::call_arguments::compose_body(
&request.optional_params,
&body,
request.config.get_supported_ocr_params(&request.model),
@ -191,10 +193,9 @@ pub(crate) fn prepare_request_for_test(request: ResolvedOcrRequest) -> PreparedO
#[cfg(test)]
mod tests {
use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
use serde_json::json;
use crate::call_arguments::{CallArguments, compose_body, parse_options};
#[derive(serde::Deserialize)]
struct KnownParams {
pages: Option<Vec<i64>>,

28
litellm-rust/crates/core/src/ocr/provider_config.rs
View file

@ -1,3 +1,4 @@
use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
use strum::{EnumString, IntoStaticStr};
use super::{
@ -7,21 +8,18 @@ use super::{
ResolvedOcrCredentials,
},
};
use crate::{
litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider},
llms::{
azure_ai::ocr::{
cohere_parse_transformation::AzureAICohereParseConfig,
document_intelligence::transformation::AzureDocumentIntelligenceOcrConfig,
transformation::AzureAiOcrConfig,
},
base_llm::ocr::transformation::{BaseOcrConfig, OcrResponseContext},
cohere::ocr::transformation::CohereParseConfig,
mistral::ocr::transformation::MistralOcrConfig,
reducto::ocr::transformation::{ReductoParseLegacyConfig, ReductoParseV3Config},
vertex_ai::ocr::{
deepseek_transformation::VertexAIDeepSeekOCRConfig, transformation::VertexAiOcrConfig,
},
use crate::llms::{
azure_ai::ocr::{
cohere_parse_transformation::AzureAICohereParseConfig,
document_intelligence::transformation::AzureDocumentIntelligenceOcrConfig,
transformation::AzureAiOcrConfig,
},
base_llm::ocr::transformation::{BaseOcrConfig, OcrResponseContext},
cohere::ocr::transformation::CohereParseConfig,
mistral::ocr::transformation::MistralOcrConfig,
reducto::ocr::transformation::{ReductoParseLegacyConfig, ReductoParseV3Config},
vertex_ai::ocr::{
deepseek_transformation::VertexAIDeepSeekOCRConfig, transformation::VertexAiOcrConfig,
},
};

10
litellm-rust/crates/core/src/ocr/types.rs
View file

@ -2,16 +2,16 @@ use std::{collections::BTreeMap, path::PathBuf, time::Duration};
use bytes::Bytes;
use litellm_auth::{InputSource, Sourced, TokenProviderHandle};
use litellm_core_utils::{
call_arguments::CallArguments,
serde_compat::{FiniteF64, LaxI64},
};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use serde_with::serde_as;
use super::provider_config::{OcrConfigKind, resolve_provider_config};
use crate::{
call_arguments::CallArguments,
constants::OCR_HTTP_TIMEOUT_SECS,
serde_compat::{FiniteF64, LaxI64},
};
use crate::constants::OCR_HTTP_TIMEOUT_SECS;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(tag = "type")]

4
litellm-rust/crates/core/src/ocr/wire.rs
View file

@ -10,11 +10,11 @@ use super::{Error, LiteLLMOcrRequest, OcrConnectionInputs, OcrDocument, OcrDocum
pub fn consumed_optional_params(
model: &str,
provider: Option<&str>,
) -> Result<Vec<crate::call_arguments::ArgumentSpec>, Error> {
) -> Result<Vec<litellm_core_utils::call_arguments::ArgumentSpec>, Error> {
let specs = super::consumed_optional_params(model, provider)?;
Ok(consumed_optional_param_names(model, provider)?
.into_iter()
.map(|name| crate::call_arguments::ArgumentSpec {
.map(|name| litellm_core_utils::call_arguments::ArgumentSpec {
name,
secret: specs.iter().any(|spec| spec.name == name && spec.secret),
})

1
litellm-rust/crates/core/src/responses/mod.rs
View file

@ -1,4 +1,3 @@
mod error;
pub use error::Error;
pub mod types;
pub mod websocket;

8
litellm-rust/crates/core/src/responses/websocket.rs
View file

@ -6,6 +6,9 @@ use std::{
};
use futures_util::{SinkExt, StreamExt};
use litellm_types::responses::streaming_websocket::{
ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult,
};
use rustls::{ClientConfig, RootCertStore};
use tokio::{net::TcpStream, sync::Mutex};
use tokio_tungstenite::{
@ -20,10 +23,7 @@ use tokio_tungstenite::{
};
use super::Error;
use crate::{
constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH},
responses::types::{ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult},
};
use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH};
pub trait ResponsesWebSocketProviderConfig: Sync {
fn supports_native_websocket(&self) -> bool {

16
litellm-rust/crates/core/tests/ocr/passthrough.rs
View file

@ -1,15 +1,19 @@
use std::collections::BTreeSet;
use std::sync::{Arc, Mutex};
use std::{
collections::BTreeSet,
sync::{Arc, Mutex},
};
use litellm_callbacks::event::{RequestContext, WireRequest};
use rstest::rstest;
use rstest_reuse::{self, apply, template};
use serde_json::{Map, Value, json};
use super::LocalOcrHost;
use super::test_support::{
MockResponse, SERVED_DOCUMENT, document_server, mock_server, perform_ocr_with, request_body,
wire_request_with_document,
use super::{
LocalOcrHost,
test_support::{
MockResponse, SERVED_DOCUMENT, document_server, mock_server, perform_ocr_with,
request_body, wire_request_with_document,
},
};
#[derive(Clone, Copy, Debug)]

2
litellm-rust/crates/providers/Cargo.toml
View file

@ -6,6 +6,8 @@ license.workspace = true
repository.workspace = true
[dependencies]
litellm-types.workspace = true
litellm-core-utils.workspace = true
litellm-auth.workspace = true
litellm-auth-aws.workspace = true
serde.workspace = true

2
litellm-rust/crates/providers/src/anthropic/chat/tests.rs
View file

@ -1,7 +1,7 @@
use serde_json::json;
use super::*;
use crate::chat::Error;
use crate::base_llm::chat::transformation::Error;
fn messages(value: Value) -> Vec<ChatMessage> {
serde_json::from_value(value).expect("valid messages")

32
litellm-rust/crates/providers/src/anthropic/chat/transformation.rs
View file

@ -1,18 +1,24 @@
use litellm_core_utils::{
core_helpers::{finish_reason_for, unix_now, usage_from_parts},
prompt_templates::factory::{Conversation, build_conversation},
};
use litellm_types::{
llms::openai::ChatMessage,
utils::{ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse},
};
use serde_json::{Map, Value, json};
use crate::anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX;
use crate::anthropic::experimental_pass_through::messages::transformation::{
complete_anthropic_url, resolve_anthropic_api_key,
};
use crate::base_llm::chat::transformation::{
BaseConfig, ChatCompletionsAuth, Unsupported, unsupported_message, unsupported_param,
};
use crate::chat::Error;
use crate::chat::conversation::{Conversation, build_conversation};
use crate::chat::response_utils::{finish_reason_for, unix_now, usage_from_parts};
use crate::chat::types::{
ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, ChatMessage,
ProviderChatRequestData, ProviderChatResponseData,
use crate::{
anthropic::{
ANTHROPIC_OAUTH_TOKEN_PREFIX,
experimental_pass_through::messages::transformation::{
complete_anthropic_url, resolve_anthropic_api_key,
},
},
base_llm::chat::transformation::{
BaseConfig, ChatCompletionsAuth, Error, ProviderChatRequestData, ProviderChatResponseData,
Unsupported, unsupported_message, unsupported_param,
},
};
/// Anthropic parameter names, post `map_openai_params`, that the Rust path can

5
litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/transformation.rs
View file

@ -1,5 +1,6 @@
use crate::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig;
use crate::messages::Error;
use crate::base_llm::{
anthropic_messages::transformation::BaseAnthropicMessagesConfig, chat::transformation::Error,
};
const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY";
const ANTHROPIC_API_BASE_ENV: &str = "ANTHROPIC_API_BASE";

31
litellm-rust/crates/providers/src/audio_transcription/mod.rs
View file

@ -1,31 +0,0 @@
use thiserror::Error;
#[derive(Clone, Debug, PartialEq, Eq, Error)]
pub enum Error {
#[error("expected {expected}, got {actual}")]
InvalidType {
expected: &'static str,
actual: &'static str,
},
#[error("missing required field: {0}")]
MissingField(&'static str),
#[error("invalid request: {0}")]
InvalidRequest(String),
#[error("invalid response: {0}")]
InvalidResponse(String),
#[error(transparent)]
Auth(#[from] litellm_auth::Error),
}
pub fn json_type_name(value: &serde_json::Value) -> &'static str {
match value {
serde_json::Value::Null => "null",
serde_json::Value::Bool(_) => "boolean",
serde_json::Value::Number(_) => "number",
serde_json::Value::String(_) => "string",
serde_json::Value::Array(_) => "array",
serde_json::Value::Object(_) => "object",
}
}
pub mod types;

24
litellm-rust/crates/providers/src/azure_ai/anthropic/messages_transformation.rs
View file

@ -1,15 +1,19 @@
use litellm_types::llms::anthropic_messages::{
anthropic_request::{
AnthropicMessage, AnthropicMessagesRequest, ContentBlock, MessageContent, SystemPrompt,
},
anthropic_response::AnthropicMessagesResponse,
};
use serde_json::{Map, Value};
use crate::anthropic::experimental_pass_through::messages::transformation::{
ANTHROPIC_MESSAGES_CONFIG, AnthropicMessagesConfig, non_empty,
};
use crate::base_llm::anthropic_messages::transformation::{
BaseAnthropicMessagesConfig, MessagesAuthStrategy,
};
use crate::messages::Error;
use crate::messages::types::{
AnthropicMessage, AnthropicMessagesRequest, AnthropicMessagesResponse, ContentBlock,
MessageContent, SystemPrompt,
use crate::{
anthropic::experimental_pass_through::messages::transformation::{
ANTHROPIC_MESSAGES_CONFIG, AnthropicMessagesConfig, non_empty,
},
base_llm::{
anthropic_messages::transformation::{BaseAnthropicMessagesConfig, MessagesAuthStrategy},
chat::transformation::Error,
},
};
const AZURE_API_KEY_ENV: &str = "AZURE_API_KEY";

7
litellm-rust/crates/providers/src/base_llm/anthropic_messages/transformation.rs
View file

@ -1,5 +1,8 @@
use crate::messages::Error;
use crate::messages::types::{AnthropicMessagesRequest, AnthropicMessagesResponse};
use litellm_types::llms::anthropic_messages::{
anthropic_request::AnthropicMessagesRequest, anthropic_response::AnthropicMessagesResponse,
};
use crate::base_llm::chat::transformation::Error;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum MessagesAuthStrategy {

24
litellm-rust/crates/providers/src/base_llm/audio_transcription/transformation.rs
View file

@ -1,9 +1,25 @@
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use crate::audio_transcription::Error;
use crate::audio_transcription::types::{
AudioTranscriptionRequestData, AudioTranscriptionResponseData,
};
use crate::base_llm::chat::transformation::Error;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AudioTranscriptionRequestData {
pub body: Value,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AudioTranscriptionResponseData {
pub text: String,
}
impl AudioTranscriptionResponseData {
pub fn into_json(self) -> Value {
serde_json::json!({
"text": self.text,
})
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AudioTranscriptionAuth {

39
litellm-rust/crates/providers/src/base_llm/chat/transformation.rs
View file

@ -1,10 +1,39 @@
use litellm_types::{
llms::openai::{ChatMessage, ChatMessageContent},
utils::ChatCompletionsResponse,
};
use serde_json::{Map, Value};
use crate::chat::Error;
use crate::chat::types::{
ChatCompletionsResponse, ChatMessage, ChatMessageContent, ProviderChatRequestData,
ProviderChatResponseData,
};
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum Error {
#[error("expected {expected}, got {actual}")]
InvalidType {
expected: &'static str,
actual: &'static str,
},
#[error("missing required field: {0}")]
MissingField(&'static str),
#[error("invalid request: {0}")]
InvalidRequest(String),
#[error("invalid response: {0}")]
InvalidResponse(String),
#[error("unsupported: {0}")]
Unsupported(&'static str),
#[error(transparent)]
Auth(#[from] litellm_auth::Error),
}
/// The provider-shaped request body a config produces. Named rather than a bare
/// `Value` so the transform contract stays a typed one, mirroring
/// [`crate::base_llm::audio_transcription::transformation::AudioTranscriptionRequestData`].
pub struct ProviderChatRequestData {
pub body: Value,
}
/// The raw provider response body handed back to a config for normalization.
pub struct ProviderChatResponseData {
pub body: Value,
}
pub const STREAM_PARAM: &str = "stream";

21
litellm-rust/crates/providers/src/bedrock/audio_transcription/mod.rs
View file

@ -1,15 +1,18 @@
use litellm_auth_aws::{
bedrock_model_id_and_region,
constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE},
resolve_bedrock_region,
};
use litellm_core_utils::core_helpers::json_type_name;
use serde_json::{Map, Value, json};
use crate::audio_transcription::Error;
use crate::audio_transcription::json_type_name;
use crate::audio_transcription::types::{
AudioTranscriptionRequestData, AudioTranscriptionResponseData,
use crate::base_llm::{
audio_transcription::transformation::{
AudioTranscriptionAuth, AudioTranscriptionRequestData, AudioTranscriptionResponseData,
BaseAudioTranscriptionConfig,
},
chat::transformation::Error,
};
use crate::base_llm::audio_transcription::transformation::{
AudioTranscriptionAuth, BaseAudioTranscriptionConfig,
};
use litellm_auth_aws::constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE};
use litellm_auth_aws::{bedrock_model_id_and_region, resolve_bedrock_region};
const SUPPORTED_PARAMS: &[&str] = &["language", "prompt", "temperature", "response_format"];

29
litellm-rust/crates/providers/src/bedrock/chat/converse_transformation.rs
View file

@ -1,18 +1,25 @@
use litellm_auth_aws::{
bedrock_model_id_and_region,
constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE},
resolve_bedrock_region,
};
use litellm_core_utils::{
core_helpers::{finish_reason_for, unix_now, usage_from_parts},
prompt_templates::factory::{Conversation, TurnRole, build_conversation},
};
use litellm_types::{
llms::openai::{ChatMessage, ChatMessageContent},
utils::{
ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse,
ChatCompletionsUsage,
},
};
use serde_json::{Map, Value, json};
use crate::base_llm::chat::transformation::{
BaseConfig, ChatCompletionsAuth, Unsupported, unsupported_message, unsupported_param,
BaseConfig, ChatCompletionsAuth, Error, ProviderChatRequestData, ProviderChatResponseData,
Unsupported, unsupported_message, unsupported_param,
};
use crate::chat::Error;
use crate::chat::conversation::{Conversation, TurnRole, build_conversation};
use crate::chat::response_utils::{finish_reason_for, unix_now, usage_from_parts};
use crate::chat::types::{
ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse,
ChatCompletionsUsage, ChatMessage, ChatMessageContent, ProviderChatRequestData,
ProviderChatResponseData,
};
use litellm_auth_aws::constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE};
use litellm_auth_aws::{bedrock_model_id_and_region, resolve_bedrock_region};
/// Converse parameter names, post `map_openai_params`, that the Rust path can
/// place verbatim in `inferenceConfig`.

2
litellm-rust/crates/providers/src/bedrock/chat/tests.rs
View file

@ -1,7 +1,7 @@
use serde_json::json;
use super::*;
use crate::chat::Error;
use crate::base_llm::chat::transformation::Error;
fn messages(value: Value) -> Vec<ChatMessage> {
serde_json::from_value(value).expect("valid messages")

21
litellm-rust/crates/providers/src/chat/mod.rs
View file

@ -1,21 +0,0 @@
use thiserror::Error;
pub const EMPTY_TEXT_PLACEHOLDER: &str = " ";
#[derive(Clone, Debug, PartialEq, Eq, Error)]
pub enum Error {
#[error("missing required field: {0}")]
MissingField(&'static str),
#[error("invalid request: {0}")]
InvalidRequest(String),
#[error("invalid response: {0}")]
InvalidResponse(String),
#[error("unsupported: {0}")]
Unsupported(&'static str),
#[error(transparent)]
Auth(#[from] litellm_auth::Error),
}
pub mod conversation;
pub mod response_utils;
pub mod types;

202
litellm-rust/crates/providers/src/chat/types.rs
View file

@ -1,202 +0,0 @@
use std::time::Duration;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use crate::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth};
/// A `/chat/completions` call as it crosses into the core.
///
/// `optional_params` arrives already mapped to the provider's own parameter
/// names by the host, exactly as the messages route receives an already
/// Anthropic-shaped body. The core owns the conversation translation, the
/// provider call, and the response normalization.
pub struct ChatCompletionsRequest<'a> {
pub model: &'a str,
pub messages: Value,
pub optional_params: Map<String, Value>,
pub api_key: Option<&'a str>,
pub api_base: Option<&'a str>,
pub custom_llm_provider: Option<&'a str>,
pub extra_headers: Option<Map<String, Value>>,
pub timeout: Option<Duration>,
}
pub struct ResolvedChatCompletionsRequest<'a> {
pub model: String,
pub config: &'static dyn BaseConfig,
pub messages: Vec<ChatMessage>,
pub optional_params: Map<String, Value>,
pub api_key: Option<&'a str>,
pub api_base: Option<&'a str>,
pub extra_headers: Option<Map<String, Value>>,
pub timeout: Option<Duration>,
}
pub struct ProviderChatCompletionsRequest {
pub model: String,
pub config: &'static dyn BaseConfig,
pub url: String,
pub body: Value,
pub upstream_headers: Vec<(String, String)>,
pub auth: ChatCompletionsAuth,
pub optional_params: Map<String, Value>,
pub timeout: Option<Duration>,
}
/// The provider-shaped request body a config produces. Named rather than a bare
/// `Value` so the transform contract stays a typed one, mirroring
/// [`crate::audio_transcription::types::AudioTranscriptionRequestData`].
pub struct ProviderChatRequestData {
pub body: Value,
}
/// The raw provider response body handed back to a config for normalization.
pub struct ProviderChatResponseData {
pub body: Value,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum ChatMessageContent {
Text(String),
Parts(Vec<Value>),
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ChatMessage {
pub role: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub content: Option<ChatMessageContent>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
/// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python
/// path reports so cost tracking sees the same numbers on either path.
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct PromptTokensDetails {
pub cached_tokens: u64,
pub cache_creation_tokens: u64,
pub text_tokens: u64,
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct ChatCompletionsUsage {
pub prompt_tokens: u64,
pub completion_tokens: u64,
pub total_tokens: u64,
pub prompt_tokens_details: PromptTokensDetails,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ChatCompletionsChoiceMessage {
pub role: String,
// Whether an empty turn is `None` or `""` is the provider's choice, not a
// shared invariant: Anthropic's transform ends on `merged_text or None`
// while Converse assigns the joined string unconditionally. Each config
// mirrors its own, so keep this optional and serialize it even when None.
pub content: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ChatCompletionsChoice {
pub index: u64,
pub message: ChatCompletionsChoiceMessage,
pub finish_reason: String,
}
/// The normalized response handed back to the host.
///
/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the
/// `ModelResponse` it already created, and echoing the provider's own id here
/// would change it. Pinned by `response_carries_no_id` in `tests.rs`.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ChatCompletionsResponse {
pub created: u64,
pub model: String,
pub choices: Vec<ChatCompletionsChoice>,
pub usage: ChatCompletionsUsage,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ChatCompletionToolCallFunctionChunk {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
pub arguments: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_specific_fields: Option<Map<String, Value>>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ChatCompletionToolCallChunk {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
#[serde(rename = "type")]
pub tool_type: String,
pub function: ChatCompletionToolCallFunctionChunk,
pub index: i64,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ChatCompletionThinkingBlock {
Thinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
thinking: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
signature: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
cache_control: Option<Value>,
},
RedactedThinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
data: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
cache_control: Option<Value>,
},
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
pub struct ChatCompletionDelta {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub content: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tool_calls: Option<Vec<ChatCompletionToolCallChunk>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reasoning_content: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub thinking_blocks: Option<Vec<ChatCompletionThinkingBlock>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_specific_fields: Option<Map<String, Value>>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ChatCompletionStreamingChoice {
pub index: u64,
pub delta: ChatCompletionDelta,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub finish_reason: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub logprobs: Option<Value>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ChatCompletionChunk {
pub id: String,
pub created: u64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub model: Option<String>,
pub object: String,
pub choices: Vec<ChatCompletionStreamingChoice>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub usage: Option<ChatCompletionsUsage>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_specific_fields: Option<Map<String, Value>>,
}

4
litellm-rust/crates/providers/src/lib.rs
View file

@ -1,8 +1,4 @@
pub mod anthropic;
pub mod audio_transcription;
pub mod azure_ai;
pub mod base_llm;
pub mod bedrock;
pub mod chat;
pub mod messages;
pub mod provider_resolution;

17
litellm-rust/crates/providers/src/messages/mod.rs
View file

@ -1,17 +0,0 @@
use thiserror::Error;
#[derive(Clone, Debug, PartialEq, Eq, Error)]
pub enum Error {
#[error("missing required field: {0}")]
MissingField(&'static str),
#[error("invalid request: {0}")]
InvalidRequest(String),
#[error("invalid response: {0}")]
InvalidResponse(String),
#[error("unsupported: {0}")]
Unsupported(&'static str),
#[error(transparent)]
Auth(#[from] litellm_auth::Error),
}
pub mod types;

33
litellm-rust/crates/providers/src/provider_resolution.rs
View file

@ -1,33 +0,0 @@
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct CustomLlmProvider<'a> {
pub model: &'a str,
pub custom_llm_provider: &'a str,
}
pub fn get_custom_llm_provider<'a>(
model: &'a str,
custom_llm_provider: Option<&'a str>,
) -> Option<CustomLlmProvider<'a>> {
if let Some(custom_llm_provider) = custom_llm_provider.filter(|provider| !provider.is_empty()) {
return Some(CustomLlmProvider {
model: strip_custom_llm_provider_prefix(model, custom_llm_provider),
custom_llm_provider,
});
}
let (custom_llm_provider, model) = model.split_once('/')?;
if custom_llm_provider.is_empty() || model.is_empty() {
return None;
}
Some(CustomLlmProvider {
model,
custom_llm_provider,
})
}
fn strip_custom_llm_provider_prefix<'a>(model: &'a str, custom_llm_provider: &str) -> &'a str {
model
.strip_prefix(custom_llm_provider)
.and_then(|model| model.strip_prefix('/'))
.unwrap_or(model)
}

1
litellm-rust/crates/python-bridge/Cargo.toml
View file

@ -20,6 +20,7 @@ bytes.workspace = true
litellm-auth.workspace = true
litellm-callbacks-legacy.workspace = true
litellm-core.workspace = true
litellm-types.workspace = true
litellm-host-python.workspace = true
litellm-token-counter.workspace = true
pyo3.workspace = true

6
litellm-rust/crates/python-bridge/benches/serialization.rs
View file

@ -1,10 +1,8 @@
use std::hint::black_box;
use std::time::Duration;
use std::{hint::black_box, time::Duration};
use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main};
use litellm_host_python::{from_py, to_py};
use pyo3::prelude::*;
use pyo3::types::PyDict;
use pyo3::{prelude::*, types::PyDict};
use serde_json::{Value, json};
const PAYLOAD_SIZES: &[(&str, usize)] = &[

10
litellm-rust/crates/python-bridge/src/credentials.rs
View file

@ -3,10 +3,12 @@
use litellm_auth::{ResolvedCredential, SecretValue};
use litellm_host_python::wrap_failure;
use pyo3::exceptions::PyTypeError;
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyString};
use pyo3::{
exceptions::PyTypeError,
gc::{PyTraverseError, PyVisit},
prelude::*,
types::{PyDict, PyString},
};
const NOT_CALLABLE: &str = "Azure AD token provider must be callable";
const NOT_A_STRING: &str = "Azure AD token must be a string, got {}";

3
litellm-rust/crates/python-bridge/src/diagnostics.rs
View file

@ -1,6 +1,5 @@
use litellm_host_python::release_count;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use pyo3::{prelude::*, types::PyDict};
#[pyfunction]
pub(crate) fn gil_stats(py: Python<'_>) -> PyResult<Py<PyAny>> {

12
litellm-rust/crates/python-bridge/src/errors.rs
View file

@ -1,7 +1,11 @@
use litellm_core::transport::Error as TransportError;
use litellm_core::{Error, audio_transcription, chat_completions, messages, ocr, responses};
use pyo3::exceptions::{PyRuntimeError, PyValueError};
use pyo3::prelude::*;
use litellm_core::{
Error, audio_transcription, chat_completions, messages, ocr, responses,
transport::Error as TransportError,
};
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
prelude::*,
};
pyo3::create_exception!(
_native,

16
litellm-rust/crates/python-bridge/src/marshal.rs
View file

@ -1,13 +1,12 @@
use std::collections::{BTreeMap, HashMap};
use std::time::Duration;
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use serde_json::{Map, Value};
use std::{
collections::{BTreeMap, HashMap},
time::Duration,
};
use litellm_auth::InputSource;
use litellm_host_python::{from_py, from_py_argument};
use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict};
use serde_json::{Map, Value};
/// The keyword arguments every value route shares, validated at the Python boundary.
pub(crate) struct RouteOptions {
@ -156,10 +155,11 @@ pub(crate) fn marshal_headers(headers: Option<Value>) -> PyResult<HashMap<String
#[cfg(test)]
mod tests {
use super::*;
use pyo3::exceptions::PyTypeError;
use serde_json::json;
use super::*;
fn eval<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> {
let locals = PyDict::new(py);
py.run(source, Some(&locals), Some(&locals)).unwrap();

8
litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs
View file

@ -1,13 +1,13 @@
use litellm_core::audio_transcription::{
AudioTranscriptionRequest, Error, audio_transcription as run_audio_transcription,
Error, audio_transcription as run_audio_transcription, types::AudioTranscriptionRequest,
};
use litellm_host_python::{from_py_argument, run_async, run_sync};
use pyo3::prelude::*;
use serde_json::{Map, Value};
use crate::errors::audio_transcription_error_to_pyerr;
use crate::marshal::{
RouteOptions, extra_headers_argument, optional_params_argument, optional_timeout,
use crate::{
errors::audio_transcription_error_to_pyerr,
marshal::{RouteOptions, extra_headers_argument, optional_params_argument, optional_timeout},
};
async fn execute(

19
litellm-rust/crates/python-bridge/src/routes/chat_completions.rs
View file

@ -1,16 +1,18 @@
use litellm_core::chat_completions::Error;
use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompletionsResponse};
use litellm_core::chat_completions::{
chat_completions as run_chat_completions, chat_completions_decline_reason,
Error, chat_completions as run_chat_completions, chat_completions_decline_reason,
types::ChatCompletionsRequest,
};
use litellm_host_python::{from_py_argument, run_async, run_sync};
use litellm_types::utils::ChatCompletionsResponse;
use pyo3::prelude::*;
use serde_json::{Map, Value};
use crate::errors::chat_completions_error_to_pyerr;
use crate::marshal::{
RouteOptions, extra_headers_argument, messages_argument, optional_params_argument,
optional_timeout,
use crate::{
errors::chat_completions_error_to_pyerr,
marshal::{
RouteOptions, extra_headers_argument, messages_argument, optional_params_argument,
optional_timeout,
},
};
async fn execute(
@ -122,8 +124,7 @@ pub(crate) fn achat_completions<'py>(
#[cfg(test)]
mod tests {
use pyo3::prelude::*;
use pyo3::types::PyList;
use pyo3::{prelude::*, types::PyList};
#[test]
fn chat_completions_decline_keeps_existing_reasons() {

11
litellm-rust/crates/python-bridge/src/routes/messages.rs
View file

@ -1,12 +1,13 @@
use litellm_core::messages::Error;
use litellm_core::messages::messages as run_messages;
use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest};
use litellm_core::messages::{Error, messages as run_messages, types::MessagesRequest};
use litellm_host_python::{run_async, run_sync};
use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse;
use pyo3::prelude::*;
use serde_json::{Map, Value};
use crate::errors::messages_error_to_pyerr;
use crate::marshal::{RouteOptions, body_argument, extra_headers_argument, optional_timeout};
use crate::{
errors::messages_error_to_pyerr,
marshal::{RouteOptions, body_argument, extra_headers_argument, optional_timeout},
};
async fn execute(
body: Map<String, Value>,

6
litellm-rust/crates/python-bridge/src/routes/mod.rs
View file

@ -6,8 +6,10 @@ pub(crate) mod responses;
#[cfg(test)]
mod tests {
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyList};
use pyo3::{
prelude::*,
types::{PyDict, PyList},
};
#[test]
fn sync_and_async_route_signatures_match_the_python_contract() {

16
litellm-rust/crates/python-bridge/src/routes/ocr/document.rs
View file

@ -1,13 +1,14 @@
use std::path::PathBuf;
use bytes::Bytes;
use pyo3::exceptions::{PyTypeError, PyValueError};
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use pyo3::pybacked::PyBackedBytes;
use pyo3::types::{PyBytes, PyString};
use litellm_core::ocr::{OcrDocumentInput, OcrFileContent};
use pyo3::{
exceptions::{PyTypeError, PyValueError},
gc::{PyTraverseError, PyVisit},
prelude::*,
pybacked::PyBackedBytes,
types::{PyBytes, PyString},
};
#[derive(Debug)]
pub(super) struct PythonFileReader {
@ -128,9 +129,10 @@ impl FromPyObject<'_, '_> for FileDocumentInput {
#[cfg(test)]
mod tests {
use super::*;
use pyo3::types::PyDict;
use super::*;
fn eval<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> {
let locals = PyDict::new(py);
py.run(source, Some(&locals), Some(&locals)).unwrap();

9
litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs
View file

@ -1,6 +1,8 @@
use litellm_core::ocr::Error;
use pyo3::exceptions::{PyFileNotFoundError, PyOSError};
use pyo3::prelude::*;
use pyo3::{
exceptions::{PyFileNotFoundError, PyOSError},
prelude::*,
};
use crate::errors::{RustUpstreamError, core_error_to_pyerr};
@ -59,9 +61,10 @@ fn attach_status(error: PyErr, status: Option<u16>) -> PyErr {
#[cfg(test)]
mod tests {
use super::*;
use pyo3::exceptions::PyValueError;
use super::*;
#[test]
fn preserves_python_validation_and_provider_details() {
Python::initialize();

16
litellm-rust/crates/python-bridge/src/routes/ocr/host.rs
View file

@ -1,13 +1,17 @@
use litellm_auth::ResolvedCredential;
use litellm_core::ocr::{LiteLLMOcrResponse, Ocr, OcrOp, OcrOpResult};
use litellm_host_python::{RouteHost, missing_state, to_py};
use pyo3::exceptions::PyBaseException;
use pyo3::gc::{PyTraverseError, PyVisit};
use pyo3::prelude::*;
use pyo3::types::PyDict;
use pyo3::{
exceptions::PyBaseException,
gc::{PyTraverseError, PyVisit},
prelude::*,
types::PyDict,
};
use super::errors::to_pyerr as ocr_error_to_pyerr;
use super::project::{OcrHostHandles, project_request};
use super::{
errors::to_pyerr as ocr_error_to_pyerr,
project::{OcrHostHandles, project_request},
};
enum OcrHostData {
Unprojected,

9
litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs
View file

@ -3,12 +3,13 @@ mod errors;
mod host;
mod project;
use host::OcrRouteHost;
use litellm_callbacks_legacy::{LegacySurface, PublicCall, run_legacy_call};
use litellm_core::ocr::{OcrClient, ocr_machine};
use pyo3::prelude::*;
use pyo3::types::{PyDict, PyTuple};
use host::OcrRouteHost;
use pyo3::{
prelude::*,
types::{PyDict, PyTuple},
};
const SURFACE: LegacySurface = LegacySurface {
call_type: "ocr",

22
litellm-rust/crates/python-bridge/src/routes/ocr/project.rs
View file

@ -1,17 +1,19 @@
use litellm_core::ocr::wire::{
OcrWireRequest, consumed_optional_params, decode_document, decode_request_input,
use litellm_core::ocr::{
LiteLLMOcrRequest, OcrDocumentInput,
wire::{OcrWireRequest, consumed_optional_params, decode_document, decode_request_input},
};
use litellm_core::ocr::{LiteLLMOcrRequest, OcrDocumentInput};
use litellm_host_python::from_py;
use pyo3::exceptions::PyValueError;
use pyo3::prelude::*;
use pyo3::types::PyDict;
use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict};
use serde_json::{Map, Value};
use super::document::{FileDocumentInput, PythonFileReader};
use super::errors::to_pyerr as ocr_error_to_pyerr;
use crate::credentials::{self, CallerTokenProvider};
use crate::marshal::{project_optional_fields, python_timeout_seconds, request_input_sources};
use super::{
document::{FileDocumentInput, PythonFileReader},
errors::to_pyerr as ocr_error_to_pyerr,
};
use crate::{
credentials::{self, CallerTokenProvider},
marshal::{project_optional_fields, python_timeout_seconds, request_input_sources},
};
/// What the host keeps after projection: the caller's callables that answer the document
/// read and token operations, and the provider name the failure mapping reports.

12
litellm-rust/crates/python-bridge/src/routes/responses.rs
View file

@ -2,8 +2,10 @@ use litellm_core::responses::websocket::ResponsesWebSocketConnection as RustResp
use pyo3::prelude::*;
use serde_json::Value;
use crate::errors::responses_error_to_pyerr;
use crate::marshal::{marshal_headers, optional_timeout};
use crate::{
errors::responses_error_to_pyerr,
marshal::{marshal_headers, optional_timeout},
};
#[pyclass]
pub(crate) struct ResponsesWebSocketConnection {
@ -58,12 +60,10 @@ impl ResponsesWebSocketConnection {
#[cfg(test)]
mod tests {
use std::ffi::CString;
use std::time::Duration;
use std::{ffi::CString, time::Duration};
use futures_util::{SinkExt, StreamExt};
use pyo3::prelude::*;
use pyo3::types::PyDict;
use pyo3::{prelude::*, types::PyDict};
use tokio::net::TcpListener;
use tokio_tungstenite::{accept_async, tungstenite::Message};

15
litellm-rust/crates/python-bridge/src/token_counter.rs
View file

@ -1,18 +1,17 @@
use std::num::NonZero;
use std::sync::Arc;
use std::thread::available_parallelism;
use std::{num::NonZero, sync::Arc, thread::available_parallelism};
use litellm_host_python::release_gil;
use litellm_host_python::{release_gil, run_async};
use litellm_token_counter::{
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
};
use pyo3::exceptions::{PyRuntimeError, PyValueError};
use pyo3::prelude::*;
use pyo3::types::PyAny;
use pyo3::{
exceptions::{PyRuntimeError, PyValueError},
prelude::*,
types::PyAny,
};
use tokio::sync::Semaphore;
use crate::errors::RustBridgeDeclined;
use litellm_host_python::run_async;
/// Counts the input tokens of a raw request body off the Python event loop with
/// the GIL released. Python owns which requests get here and what to do with

6
litellm-rust/crates/python-bridge/tests/marshal_boundary.rs
View file

@ -1,5 +1,7 @@
use std::fs;
use std::path::{Path, PathBuf};
use std::{
fs,
path::{Path, PathBuf},
};
const DISALLOWED_OUTSIDE_INTEROP: &[&str] = &[
"py.import(\"json\")",

10
litellm-rust/crates/types/Cargo.toml Normal file
View file

@ -0,0 +1,10 @@
[package]
name = "litellm-types"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
serde.workspace = true
serde_json.workspace = true

3
litellm-rust/crates/types/src/lib.rs Normal file
View file

@ -0,0 +1,3 @@
pub mod llms;
pub mod responses;
pub mod utils;

44
litellm-rust/crates/providers/src/messages/types.rs → litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_request.rs
View file

@ -1,30 +1,6 @@
use std::time::Duration;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use crate::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig;
pub struct MessagesRequest<'a> {
pub model: &'a str,
pub body: Value,
pub api_key: Option<&'a str>,
pub api_base: Option<&'a str>,
pub custom_llm_provider: Option<&'a str>,
pub extra_headers: Option<Map<String, Value>>,
pub timeout: Option<Duration>,
}
pub struct ProviderMessagesRequest {
pub provider: String,
pub model: String,
pub config: &'static dyn BaseAnthropicMessagesConfig,
pub url: String,
pub body: Value,
pub upstream_headers: Vec<(String, String)>,
pub timeout: Option<Duration>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum SystemPrompt {
@ -112,23 +88,3 @@ pub struct AnthropicMessagesRequest {
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AnthropicMessagesResponse {
pub id: String,
#[serde(rename = "type")]
pub message_type: String,
pub role: String,
pub model: String,
pub content: Vec<Value>,
// Anthropic always includes stop_reason / stop_sequence, null until the turn
// ends; serialize them even when None so callers see the same shape as Python.
pub stop_reason: Option<String>,
pub stop_sequence: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub usage: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub container: Option<Value>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

22
litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_response.rs Normal file
View file

@ -0,0 +1,22 @@
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct AnthropicMessagesResponse {
pub id: String,
#[serde(rename = "type")]
pub message_type: String,
pub role: String,
pub model: String,
pub content: Vec<Value>,
// Anthropic always includes stop_reason / stop_sequence, null until the turn
// ends; serialize them even when None so callers see the same shape as Python.
pub stop_reason: Option<String>,
pub stop_sequence: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
pub usage: Option<Value>,
#[serde(skip_serializing_if = "Option::is_none")]
pub container: Option<Value>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}

2
litellm-rust/crates/types/src/llms/anthropic_messages/mod.rs Normal file
View file

@ -0,0 +1,2 @@
pub mod anthropic_request;
pub mod anthropic_response;

2
litellm-rust/crates/types/src/llms/mod.rs Normal file
View file

@ -0,0 +1,2 @@
pub mod anthropic_messages;
pub mod openai;

58
litellm-rust/crates/types/src/llms/openai.rs Normal file
View file

@ -0,0 +1,58 @@
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(untagged)]
pub enum ChatMessageContent {
Text(String),
Parts(Vec<Value>),
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ChatMessage {
pub role: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub content: Option<ChatMessageContent>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ChatCompletionToolCallFunctionChunk {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
pub arguments: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_specific_fields: Option<Map<String, Value>>,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct ChatCompletionToolCallChunk {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub id: Option<String>,
#[serde(rename = "type")]
pub tool_type: String,
pub function: ChatCompletionToolCallFunctionChunk,
pub index: i64,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ChatCompletionThinkingBlock {
Thinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
thinking: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
signature: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
cache_control: Option<Value>,
},
RedactedThinking {
#[serde(default, skip_serializing_if = "Option::is_none")]
data: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
cache_control: Option<Value>,
},
}

1
litellm-rust/crates/types/src/responses/mod.rs Normal file
View file

@ -0,0 +1 @@
pub mod streaming_websocket;

0
litellm-rust/crates/core/src/responses/types.rs → litellm-rust/crates/types/src/responses/streaming_websocket.rs
View file

Some files were not shown because too many files have changed in this diff Show more