From 904c679595fa22e726ad788ec979893d2b8af6f6 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Thu, 17 Sep 2026 22:01:14 -0700 Subject: [PATCH] refactor(rust): add types and core-utils crates, unify the provider error litellm-types mirrors litellm/types (chat, Anthropic Messages and Responses websocket data) and litellm-core-utils mirrors litellm/litellm_core_utils (provider resolution, prompt factory, core helpers, call arguments). Route request types move up to their core route module, the transform contracts move into base_llm, and the three duplicated provider error enums become one Error in base_llm/chat/transformation.rs, mirroring BaseLLMException. The empty-text placeholder goes back to the value Python's factory.py uses; the provider extraction had changed it to a single space. Co-Authored-By: Claude Opus 5 --- litellm-rust/Cargo.lock | 27 +++ litellm-rust/Cargo.toml | 2 + litellm-rust/crates/core-utils/Cargo.toml | 18 ++ .../src/call_arguments.rs | 2 +- .../src/core_helpers.rs} | 13 +- .../src}/get_llm_provider_logic.rs | 34 ++- litellm-rust/crates/core-utils/src/lib.rs | 7 + .../crates/{core => core-utils}/src/params.rs | 0 .../src/prompt_templates/factory.rs} | 12 +- .../core-utils/src/prompt_templates/mod.rs | 1 + .../{core => core-utils}/src/serde_compat.rs | 4 +- .../{core => core-utils}/src/url_utils.rs | 19 +- litellm-rust/crates/core/Cargo.toml | 2 + .../core/src/audio_transcription/error.rs | 25 +-- .../core/src/audio_transcription/handler.rs | 7 +- .../core/src/audio_transcription/mod.rs | 5 +- .../core/src/audio_transcription/prepare.rs | 8 +- .../core/src/audio_transcription/tests.rs | 3 +- .../src/audio_transcription/types.rs | 24 +-- .../crates/core/src/chat_completions/error.rs | 21 +- .../core/src/chat_completions/handler.rs | 18 +- .../crates/core/src/chat_completions/mod.rs | 7 +- .../core/src/chat_completions/prepare.rs | 10 +- .../crates/core/src/chat_completions/tests.rs | 2 +- .../crates/core/src/chat_completions/types.rs | 44 ++++ litellm-rust/crates/core/src/http_utils.rs | 12 +- litellm-rust/crates/core/src/lib.rs | 5 - .../crates/core/src/litellm_core_utils/mod.rs | 1 - .../core/src/llms/anthropic/chat/streaming.rs | 13 +- .../messages/batches.rs | 3 +- .../messages/count_tokens.rs | 11 +- .../ocr/cohere_parse_transformation.rs | 3 +- .../document_intelligence/transformation.rs | 8 +- .../src/llms/azure_ai/ocr/transformation.rs | 40 ++-- .../src/llms/base_llm/ocr/transformation.rs | 16 +- .../src/llms/cohere/ocr/transformation.rs | 8 +- .../src/llms/mistral/ocr/transformation.rs | 4 +- .../llms/openai/responses/transformation.rs | 3 +- .../src/llms/reducto/ocr/transformation.rs | 17 +- .../vertex_ai/ocr/deepseek_transformation.rs | 14 +- .../src/llms/vertex_ai/ocr/transformation.rs | 4 +- .../crates/core/src/messages/error.rs | 21 +- .../crates/core/src/messages/handler.rs | 12 +- litellm-rust/crates/core/src/messages/mod.rs | 6 +- .../crates/core/src/messages/prepare.rs | 6 +- .../crates/core/src/messages/tests.rs | 2 +- .../crates/core/src/messages/types.rs | 24 +++ litellm-rust/crates/core/src/ocr/arguments.rs | 3 +- litellm-rust/crates/core/src/ocr/error.rs | 6 +- litellm-rust/crates/core/src/ocr/prepare.rs | 13 +- .../crates/core/src/ocr/provider_config.rs | 28 ++- litellm-rust/crates/core/src/ocr/types.rs | 10 +- litellm-rust/crates/core/src/ocr/wire.rs | 4 +- litellm-rust/crates/core/src/responses/mod.rs | 1 - .../crates/core/src/responses/websocket.rs | 8 +- .../crates/core/tests/ocr/passthrough.rs | 16 +- litellm-rust/crates/providers/Cargo.toml | 2 + .../providers/src/anthropic/chat/tests.rs | 2 +- .../src/anthropic/chat/transformation.rs | 32 +-- .../messages/transformation.rs | 5 +- .../providers/src/audio_transcription/mod.rs | 31 --- .../anthropic/messages_transformation.rs | 24 ++- .../anthropic_messages/transformation.rs | 7 +- .../audio_transcription/transformation.rs | 24 ++- .../src/base_llm/chat/transformation.rs | 39 +++- .../src/bedrock/audio_transcription/mod.rs | 21 +- .../bedrock/chat/converse_transformation.rs | 29 ++- .../providers/src/bedrock/chat/tests.rs | 2 +- litellm-rust/crates/providers/src/chat/mod.rs | 21 -- .../crates/providers/src/chat/types.rs | 202 ------------------ litellm-rust/crates/providers/src/lib.rs | 4 - .../crates/providers/src/messages/mod.rs | 17 -- .../providers/src/provider_resolution.rs | 33 --- litellm-rust/crates/python-bridge/Cargo.toml | 1 + .../python-bridge/benches/serialization.rs | 6 +- .../crates/python-bridge/src/credentials.rs | 10 +- .../crates/python-bridge/src/diagnostics.rs | 3 +- .../crates/python-bridge/src/errors.rs | 12 +- .../crates/python-bridge/src/marshal.rs | 16 +- .../src/routes/audio_transcription.rs | 8 +- .../src/routes/chat_completions.rs | 19 +- .../python-bridge/src/routes/messages.rs | 11 +- .../crates/python-bridge/src/routes/mod.rs | 6 +- .../python-bridge/src/routes/ocr/document.rs | 16 +- .../python-bridge/src/routes/ocr/errors.rs | 9 +- .../python-bridge/src/routes/ocr/host.rs | 16 +- .../python-bridge/src/routes/ocr/mod.rs | 9 +- .../python-bridge/src/routes/ocr/project.rs | 22 +- .../python-bridge/src/routes/responses.rs | 12 +- .../crates/python-bridge/src/token_counter.rs | 15 +- .../python-bridge/tests/marshal_boundary.rs | 6 +- litellm-rust/crates/types/Cargo.toml | 10 + litellm-rust/crates/types/src/lib.rs | 3 + .../anthropic_messages/anthropic_request.rs} | 44 ---- .../anthropic_messages/anthropic_response.rs | 22 ++ .../types/src/llms/anthropic_messages/mod.rs | 2 + litellm-rust/crates/types/src/llms/mod.rs | 2 + litellm-rust/crates/types/src/llms/openai.rs | 58 +++++ .../crates/types/src/responses/mod.rs | 1 + .../src/responses/streaming_websocket.rs} | 0 litellm-rust/crates/types/src/utils.rs | 93 ++++++++ 101 files changed, 815 insertions(+), 749 deletions(-) create mode 100644 litellm-rust/crates/core-utils/Cargo.toml rename litellm-rust/crates/{core => core-utils}/src/call_arguments.rs (98%) rename litellm-rust/crates/{providers/src/chat/response_utils.rs => core-utils/src/core_helpers.rs} (88%) rename litellm-rust/crates/{core/src/litellm_core_utils => core-utils/src}/get_llm_provider_logic.rs (59%) create mode 100644 litellm-rust/crates/core-utils/src/lib.rs rename litellm-rust/crates/{core => core-utils}/src/params.rs (100%) rename litellm-rust/crates/{providers/src/chat/conversation.rs => core-utils/src/prompt_templates/factory.rs} (94%) create mode 100644 litellm-rust/crates/core-utils/src/prompt_templates/mod.rs rename litellm-rust/crates/{core => core-utils}/src/serde_compat.rs (98%) rename litellm-rust/crates/{core => core-utils}/src/url_utils.rs (88%) rename litellm-rust/crates/{providers => core}/src/audio_transcription/types.rs (71%) create mode 100644 litellm-rust/crates/core/src/chat_completions/types.rs delete mode 100644 litellm-rust/crates/core/src/litellm_core_utils/mod.rs create mode 100644 litellm-rust/crates/core/src/messages/types.rs delete mode 100644 litellm-rust/crates/providers/src/audio_transcription/mod.rs delete mode 100644 litellm-rust/crates/providers/src/chat/mod.rs delete mode 100644 litellm-rust/crates/providers/src/chat/types.rs delete mode 100644 litellm-rust/crates/providers/src/messages/mod.rs delete mode 100644 litellm-rust/crates/providers/src/provider_resolution.rs create mode 100644 litellm-rust/crates/types/Cargo.toml create mode 100644 litellm-rust/crates/types/src/lib.rs rename litellm-rust/crates/{providers/src/messages/types.rs => types/src/llms/anthropic_messages/anthropic_request.rs} (69%) create mode 100644 litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_response.rs create mode 100644 litellm-rust/crates/types/src/llms/anthropic_messages/mod.rs create mode 100644 litellm-rust/crates/types/src/llms/mod.rs create mode 100644 litellm-rust/crates/types/src/llms/openai.rs create mode 100644 litellm-rust/crates/types/src/responses/mod.rs rename litellm-rust/crates/{core/src/responses/types.rs => types/src/responses/streaming_websocket.rs} (100%) create mode 100644 litellm-rust/crates/types/src/utils.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index ea98a5f6b06..196f282b5fb 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2036,8 +2036,10 @@ dependencies = [ "litellm-auth-azure", "litellm-auth-gcp", "litellm-callbacks", + "litellm-core-utils", "litellm-framing", "litellm-providers", + "litellm-types", "mime_guess", "moka", "rand 0.8.7", @@ -2061,6 +2063,20 @@ dependencies = [ "veil", ] +[[package]] +name = "litellm-core-utils" +version = "0.1.0" +dependencies = [ + "litellm-types", + "rstest", + "serde", + "serde_json", + "serde_path_to_error", + "serde_with", + "thiserror 2.0.19", + "url", +] + [[package]] name = "litellm-framing" version = "0.1.0" @@ -2096,6 +2112,8 @@ version = "0.1.0" dependencies = [ "litellm-auth", "litellm-auth-aws", + "litellm-core-utils", + "litellm-types", "rstest", "serde", "serde_json", @@ -2114,6 +2132,7 @@ dependencies = [ "litellm-core", "litellm-host-python", "litellm-token-counter", + "litellm-types", "pyo3", "pyo3-async-runtimes", "rstest", @@ -2140,6 +2159,14 @@ dependencies = [ "unicode-normalization-alignments", ] +[[package]] +name = "litellm-types" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "litemap" version = "0.8.2" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 851ef91a1fb..89b217cadda 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -18,6 +18,8 @@ litellm-auth-aws = { path = "crates/auth-aws" } litellm-auth-azure = { path = "crates/auth-azure" } litellm-auth-gcp = { path = "crates/auth-gcp" } litellm-providers = { path = "crates/providers" } +litellm-types = { path = "crates/types" } +litellm-core-utils = { path = "crates/core-utils" } litellm-cache = { path = "crates/cache" } litellm-cache-memory = { path = "crates/cache-memory" } litellm-token-counter = { path = "crates/token-counter" } diff --git a/litellm-rust/crates/core-utils/Cargo.toml b/litellm-rust/crates/core-utils/Cargo.toml new file mode 100644 index 00000000000..ea6948114eb --- /dev/null +++ b/litellm-rust/crates/core-utils/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "litellm-core-utils" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +litellm-types.workspace = true +serde.workspace = true +serde_json.workspace = true +serde_path_to_error = "0.1" +serde_with.workspace = true +thiserror.workspace = true +url.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/litellm-rust/crates/core/src/call_arguments.rs b/litellm-rust/crates/core-utils/src/call_arguments.rs similarity index 98% rename from litellm-rust/crates/core/src/call_arguments.rs rename to litellm-rust/crates/core-utils/src/call_arguments.rs index eb1dcd8deb7..31fe1978f2c 100644 --- a/litellm-rust/crates/core/src/call_arguments.rs +++ b/litellm-rust/crates/core-utils/src/call_arguments.rs @@ -8,7 +8,7 @@ use serde_json::{Map, Value}; pub struct CallArguments(Map); impl CallArguments { - pub(crate) fn select(&self, names: &[&str]) -> Map { + pub fn select(&self, names: &[&str]) -> Map { self.iter() .filter(|(name, _)| names.contains(&name.as_str())) .map(|(name, value)| (name.clone(), value.clone())) diff --git a/litellm-rust/crates/providers/src/chat/response_utils.rs b/litellm-rust/crates/core-utils/src/core_helpers.rs similarity index 88% rename from litellm-rust/crates/providers/src/chat/response_utils.rs rename to litellm-rust/crates/core-utils/src/core_helpers.rs index 1ada5d43980..cc9fc7a6687 100644 --- a/litellm-rust/crates/providers/src/chat/response_utils.rs +++ b/litellm-rust/crates/core-utils/src/core_helpers.rs @@ -2,7 +2,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; -use super::types::{ChatCompletionsUsage, PromptTokensDetails}; +use litellm_types::utils::{ChatCompletionsUsage, PromptTokensDetails}; /// OpenAI finish reasons, mirroring Python's `_FINISH_REASON_MAP` for the /// reasons the providers on this route can emit. Python warns and falls back to @@ -54,6 +54,17 @@ pub fn unix_now() -> u64 { .map_or(0, |elapsed| elapsed.as_secs()) } +pub fn json_type_name(value: &serde_json::Value) -> &'static str { + match value { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "boolean", + serde_json::Value::Number(_) => "number", + serde_json::Value::String(_) => "string", + serde_json::Value::Array(_) => "array", + serde_json::Value::Object(_) => "object", + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/litellm-rust/crates/core/src/litellm_core_utils/get_llm_provider_logic.rs b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs similarity index 59% rename from litellm-rust/crates/core/src/litellm_core_utils/get_llm_provider_logic.rs rename to litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs index 5958e8ac613..6333eedebfc 100644 --- a/litellm-rust/crates/core/src/litellm_core_utils/get_llm_provider_logic.rs +++ b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs @@ -1,4 +1,36 @@ -pub use litellm_providers::provider_resolution::{CustomLlmProvider, get_custom_llm_provider}; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CustomLlmProvider<'a> { + pub model: &'a str, + pub custom_llm_provider: &'a str, +} + +pub fn get_custom_llm_provider<'a>( + model: &'a str, + custom_llm_provider: Option<&'a str>, +) -> Option> { + if let Some(custom_llm_provider) = custom_llm_provider.filter(|provider| !provider.is_empty()) { + return Some(CustomLlmProvider { + model: strip_custom_llm_provider_prefix(model, custom_llm_provider), + custom_llm_provider, + }); + } + + let (custom_llm_provider, model) = model.split_once('/')?; + if custom_llm_provider.is_empty() || model.is_empty() { + return None; + } + Some(CustomLlmProvider { + model, + custom_llm_provider, + }) +} + +fn strip_custom_llm_provider_prefix<'a>(model: &'a str, custom_llm_provider: &str) -> &'a str { + model + .strip_prefix(custom_llm_provider) + .and_then(|model| model.strip_prefix('/')) + .unwrap_or(model) +} #[cfg(test)] mod tests { diff --git a/litellm-rust/crates/core-utils/src/lib.rs b/litellm-rust/crates/core-utils/src/lib.rs new file mode 100644 index 00000000000..a8895cccf2c --- /dev/null +++ b/litellm-rust/crates/core-utils/src/lib.rs @@ -0,0 +1,7 @@ +pub mod call_arguments; +pub mod core_helpers; +pub mod get_llm_provider_logic; +pub mod params; +pub mod prompt_templates; +pub mod serde_compat; +pub mod url_utils; diff --git a/litellm-rust/crates/core/src/params.rs b/litellm-rust/crates/core-utils/src/params.rs similarity index 100% rename from litellm-rust/crates/core/src/params.rs rename to litellm-rust/crates/core-utils/src/params.rs diff --git a/litellm-rust/crates/providers/src/chat/conversation.rs b/litellm-rust/crates/core-utils/src/prompt_templates/factory.rs similarity index 94% rename from litellm-rust/crates/providers/src/chat/conversation.rs rename to litellm-rust/crates/core-utils/src/prompt_templates/factory.rs index 587b7ea2a16..2c4921d26be 100644 --- a/litellm-rust/crates/providers/src/chat/conversation.rs +++ b/litellm-rust/crates/core-utils/src/prompt_templates/factory.rs @@ -10,8 +10,10 @@ //! `_bedrock_converse_messages_pt` for the text-only surface this route //! accepts; anything richer is declined upstream by the capability gate. -use super::types::{ChatMessage, ChatMessageContent}; -use crate::chat::EMPTY_TEXT_PLACEHOLDER; +use litellm_types::llms::openai::{ChatMessage, ChatMessageContent}; + +pub const EMPTY_TEXT_PLACEHOLDER: &str = + "[System: Empty message content sanitised to satisfy protocol]"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum TurnRole { @@ -203,8 +205,10 @@ mod tests { {"role": "assistant", "content": " "}, {"role": "user", "content": "real"} ]))); - assert_eq!(conversation.turns[0].texts, vec![EMPTY_TEXT_PLACEHOLDER]); - assert_eq!(conversation.turns[1].texts, vec![EMPTY_TEXT_PLACEHOLDER]); + // Must equal `_EMPTY_TEXT_PLACEHOLDER` in litellm/litellm_core_utils/prompt_templates/factory.py + let placeholder = "[System: Empty message content sanitised to satisfy protocol]"; + assert_eq!(conversation.turns[0].texts, vec![placeholder]); + assert_eq!(conversation.turns[1].texts, vec![placeholder]); } #[test] diff --git a/litellm-rust/crates/core-utils/src/prompt_templates/mod.rs b/litellm-rust/crates/core-utils/src/prompt_templates/mod.rs new file mode 100644 index 00000000000..a106d20eaff --- /dev/null +++ b/litellm-rust/crates/core-utils/src/prompt_templates/mod.rs @@ -0,0 +1 @@ +pub mod factory; diff --git a/litellm-rust/crates/core/src/serde_compat.rs b/litellm-rust/crates/core-utils/src/serde_compat.rs similarity index 98% rename from litellm-rust/crates/core/src/serde_compat.rs rename to litellm-rust/crates/core-utils/src/serde_compat.rs index 3ec869b40e2..bb2648eb0be 100644 --- a/litellm-rust/crates/core/src/serde_compat.rs +++ b/litellm-rust/crates/core-utils/src/serde_compat.rs @@ -2,8 +2,8 @@ use serde::{Deserialize, Deserializer, de::Error}; use serde_json::Value; use serde_with::DeserializeAs; -pub(crate) struct LaxI64; -pub(crate) struct FiniteF64; +pub struct LaxI64; +pub struct FiniteF64; impl<'de> DeserializeAs<'de, i64> for LaxI64 { fn deserialize_as>(deserializer: D) -> Result { diff --git a/litellm-rust/crates/core/src/url_utils.rs b/litellm-rust/crates/core-utils/src/url_utils.rs similarity index 88% rename from litellm-rust/crates/core/src/url_utils.rs rename to litellm-rust/crates/core-utils/src/url_utils.rs index b8d82b7a04a..1f690752c7a 100644 --- a/litellm-rust/crates/core/src/url_utils.rs +++ b/litellm-rust/crates/core-utils/src/url_utils.rs @@ -3,33 +3,30 @@ use std::marker::PhantomData; use url::Url; #[derive(Debug, thiserror::Error)] -pub(crate) enum ApiUrlError { +pub enum ApiUrlError { #[error("invalid URL: {0}")] Parse(#[from] url::ParseError), #[error("URL cannot be used as a base")] CannotBeBase, } -pub(crate) struct Base; -pub(crate) struct Complete; +pub struct Base; +pub struct Complete; -pub(crate) struct ApiUrl { +pub struct ApiUrl { url: Url, state: PhantomData, } impl ApiUrl { - pub(crate) fn parse(value: &str) -> Result { + pub fn parse(value: &str) -> Result { Ok(Self { url: Url::parse(value.trim())?, state: PhantomData, }) } - pub(crate) fn complete_path( - mut self, - target: &[&str], - ) -> Result, ApiUrlError> { + pub fn complete_path(mut self, target: &[&str]) -> Result, ApiUrlError> { let existing: Vec = self .url .path_segments() @@ -59,7 +56,7 @@ impl ApiUrl { } impl ApiUrl { - pub(crate) fn append_query_pairs<'a>( + pub fn append_query_pairs<'a>( mut self, pairs: impl IntoIterator, ) -> Self { @@ -67,7 +64,7 @@ impl ApiUrl { self } - pub(crate) fn into_string(self) -> String { + pub fn into_string(self) -> String { self.url.into() } } diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index b9382ac7afd..33f44d66bb2 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -7,6 +7,8 @@ repository.workspace = true autotests = false [dependencies] +litellm-types.workspace = true +litellm-core-utils.workspace = true litellm-callbacks.workspace = true bytes.workspace = true futures-util.workspace = true diff --git a/litellm-rust/crates/core/src/audio_transcription/error.rs b/litellm-rust/crates/core/src/audio_transcription/error.rs index ab194173b67..cfa9e4cc5e2 100644 --- a/litellm-rust/crates/core/src/audio_transcription/error.rs +++ b/litellm-rust/crates/core/src/audio_transcription/error.rs @@ -1,3 +1,5 @@ +use litellm_providers::base_llm::chat::transformation::Error as LlmError; + #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { #[error("expected {expected}, got {actual}")] @@ -25,22 +27,15 @@ pub enum Error { Aws(#[from] litellm_auth_aws::Error), } -impl From for Error { - fn from(error: litellm_providers::audio_transcription::Error) -> Self { +impl From for Error { + fn from(error: LlmError) -> Self { match error { - litellm_providers::audio_transcription::Error::InvalidType { expected, actual } => { - Self::InvalidType { expected, actual } - } - litellm_providers::audio_transcription::Error::MissingField(field) => { - Self::MissingField(field) - } - litellm_providers::audio_transcription::Error::InvalidRequest(message) => { - Self::InvalidRequest(message) - } - litellm_providers::audio_transcription::Error::InvalidResponse(message) => { - Self::InvalidResponse(message) - } - litellm_providers::audio_transcription::Error::Auth(error) => Self::Auth(error), + LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual }, + LlmError::MissingField(field) => Self::MissingField(field), + LlmError::InvalidRequest(message) => Self::InvalidRequest(message), + LlmError::InvalidResponse(message) => Self::InvalidResponse(message), + LlmError::Unsupported(reason) => Self::Unsupported(reason), + LlmError::Auth(error) => Self::Auth(error), } } } diff --git a/litellm-rust/crates/core/src/audio_transcription/handler.rs b/litellm-rust/crates/core/src/audio_transcription/handler.rs index a7ab93ccd48..e79833cf201 100644 --- a/litellm-rust/crates/core/src/audio_transcription/handler.rs +++ b/litellm-rust/crates/core/src/audio_transcription/handler.rs @@ -1,7 +1,10 @@ use serde_json::Value; -use super::{Error, client::http_client, types::ProviderAudioTranscriptionRequest}; -use crate::http_utils::{http_request, truncate_error_body}; +use super::{Error, client::http_client}; +use crate::{ + audio_transcription::types::ProviderAudioTranscriptionRequest, + http_utils::{http_request, truncate_error_body}, +}; pub async fn execute_audio_transcription_provider_call( request: ProviderAudioTranscriptionRequest, diff --git a/litellm-rust/crates/core/src/audio_transcription/mod.rs b/litellm-rust/crates/core/src/audio_transcription/mod.rs index 5037fa2322e..af9c398c065 100644 --- a/litellm-rust/crates/core/src/audio_transcription/mod.rs +++ b/litellm-rust/crates/core/src/audio_transcription/mod.rs @@ -1,13 +1,14 @@ mod error; +pub mod types; pub use error::Error; mod client; mod handler; mod prepare; pub use handler::execute_audio_transcription_provider_call; -pub use litellm_providers::audio_transcription::types; pub use prepare::prepare_audio_transcription_provider_call; use serde_json::Value; -pub use types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; + +use crate::audio_transcription::types::AudioTranscriptionRequest; pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { execute_audio_transcription_provider_call(prepare_audio_transcription_provider_call(request)?) diff --git a/litellm-rust/crates/core/src/audio_transcription/prepare.rs b/litellm-rust/crates/core/src/audio_transcription/prepare.rs index beecdab9615..0e230117834 100644 --- a/litellm-rust/crates/core/src/audio_transcription/prepare.rs +++ b/litellm-rust/crates/core/src/audio_transcription/prepare.rs @@ -1,3 +1,4 @@ +use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; use litellm_providers::{ base_llm::audio_transcription::transformation::{ AudioTranscriptionAuth, BaseAudioTranscriptionConfig, @@ -5,13 +6,10 @@ use litellm_providers::{ bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG, }; -use super::{ - Error, - types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}, -}; +use super::Error; use crate::{ + audio_transcription::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}, http_utils::{has_header, string_headers}, - litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}, }; fn provider_config(provider: &str) -> Option<&'static dyn BaseAudioTranscriptionConfig> { diff --git a/litellm-rust/crates/core/src/audio_transcription/tests.rs b/litellm-rust/crates/core/src/audio_transcription/tests.rs index d6491ca8ce0..8ccf7a07a0f 100644 --- a/litellm-rust/crates/core/src/audio_transcription/tests.rs +++ b/litellm-rust/crates/core/src/audio_transcription/tests.rs @@ -6,7 +6,8 @@ use std::{ use serde_json::{Map, json}; -use super::{audio_transcription, types::AudioTranscriptionRequest}; +use super::audio_transcription; +use crate::audio_transcription::types::AudioTranscriptionRequest; #[tokio::test] async fn bedrock_request_is_signed_and_contains_audio() { diff --git a/litellm-rust/crates/providers/src/audio_transcription/types.rs b/litellm-rust/crates/core/src/audio_transcription/types.rs similarity index 71% rename from litellm-rust/crates/providers/src/audio_transcription/types.rs rename to litellm-rust/crates/core/src/audio_transcription/types.rs index d17d5067de5..7878e4bd175 100644 --- a/litellm-rust/crates/providers/src/audio_transcription/types.rs +++ b/litellm-rust/crates/core/src/audio_transcription/types.rs @@ -1,11 +1,9 @@ use std::time::Duration; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; - -use crate::base_llm::audio_transcription::transformation::{ +use litellm_providers::base_llm::audio_transcription::transformation::{ AudioTranscriptionAuth, BaseAudioTranscriptionConfig, }; +use serde_json::{Map, Value}; pub struct AudioTranscriptionRequest<'a> { pub model: &'a str, @@ -52,21 +50,3 @@ impl ProviderAudioTranscriptionRequest { Self { body, ..self } } } - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct AudioTranscriptionRequestData { - pub body: Value, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct AudioTranscriptionResponseData { - pub text: String, -} - -impl AudioTranscriptionResponseData { - pub fn into_json(self) -> Value { - serde_json::json!({ - "text": self.text, - }) - } -} diff --git a/litellm-rust/crates/core/src/chat_completions/error.rs b/litellm-rust/crates/core/src/chat_completions/error.rs index 95da97125d7..cfa9e4cc5e2 100644 --- a/litellm-rust/crates/core/src/chat_completions/error.rs +++ b/litellm-rust/crates/core/src/chat_completions/error.rs @@ -1,3 +1,5 @@ +use litellm_providers::base_llm::chat::transformation::Error as LlmError; + #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { #[error("expected {expected}, got {actual}")] @@ -25,18 +27,15 @@ pub enum Error { Aws(#[from] litellm_auth_aws::Error), } -impl From for Error { - fn from(error: litellm_providers::chat::Error) -> Self { +impl From for Error { + fn from(error: LlmError) -> Self { match error { - litellm_providers::chat::Error::MissingField(field) => Self::MissingField(field), - litellm_providers::chat::Error::InvalidRequest(message) => { - Self::InvalidRequest(message) - } - litellm_providers::chat::Error::InvalidResponse(message) => { - Self::InvalidResponse(message) - } - litellm_providers::chat::Error::Unsupported(reason) => Self::Unsupported(reason), - litellm_providers::chat::Error::Auth(error) => Self::Auth(error), + LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual }, + LlmError::MissingField(field) => Self::MissingField(field), + LlmError::InvalidRequest(message) => Self::InvalidRequest(message), + LlmError::InvalidResponse(message) => Self::InvalidResponse(message), + LlmError::Unsupported(reason) => Self::Unsupported(reason), + LlmError::Auth(error) => Self::Auth(error), } } } diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index d9939177f31..ae8c0dd6144 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -1,16 +1,14 @@ -use litellm_providers::base_llm::chat::transformation::ChatCompletionsAuth; +use litellm_providers::base_llm::chat::transformation::{ + ChatCompletionsAuth, ProviderChatResponseData, +}; +use litellm_types::utils::ChatCompletionsResponse; use serde_json::Value; -use super::{ - Error, - client::http_client, - prepare::prepare_provider_request, - types::{ - ChatCompletionsResponse, ProviderChatCompletionsRequest, ProviderChatResponseData, - ResolvedChatCompletionsRequest, - }, +use super::{Error, client::http_client, prepare::prepare_provider_request}; +use crate::{ + chat_completions::types::{ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest}, + http_utils::{http_request, truncate_error_body}, }; -use crate::http_utils::{http_request, truncate_error_body}; pub(super) async fn execute_chat_completions_provider_call( request: ResolvedChatCompletionsRequest<'_>, diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index 2fd619f9f93..5c5cc48692a 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -7,18 +7,19 @@ //! calls the provider, and returns a typed OpenAI-shaped response. mod error; +pub mod types; pub use error::Error; mod client; mod common_utils; -pub use litellm_providers::chat::{conversation, response_utils}; pub(crate) mod handler; mod prepare; pub mod streaming; use handler::execute_chat_completions_provider_call; -pub use litellm_providers::chat::types; +use litellm_types::utils::ChatCompletionsResponse; use prepare::{parse_messages, resolve_provider_config, resolve_request}; use serde_json::{Map, Value}; -use types::{ChatCompletionsRequest, ChatCompletionsResponse}; + +use crate::chat_completions::types::ChatCompletionsRequest; pub async fn chat_completions( request: ChatCompletionsRequest<'_>, diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/core/src/chat_completions/prepare.rs index d7b2a58596f..882da4e584b 100644 --- a/litellm-rust/crates/core/src/chat_completions/prepare.rs +++ b/litellm-rust/crates/core/src/chat_completions/prepare.rs @@ -1,17 +1,17 @@ +use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; use litellm_providers::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; +use litellm_types::llms::openai::ChatMessage; use serde_json::Value; use super::{ Error, common_utils::{chat_completions_provider_config, string_headers}, - types::{ - ChatCompletionsRequest, ChatMessage, ProviderChatCompletionsRequest, - ResolvedChatCompletionsRequest, - }, }; use crate::{ + chat_completions::types::{ + ChatCompletionsRequest, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest, + }, http_utils::has_header, - litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}, }; pub(super) fn resolve_provider_config<'a>( diff --git a/litellm-rust/crates/core/src/chat_completions/tests.rs b/litellm-rust/crates/core/src/chat_completions/tests.rs index e9f1451022e..b8bf8625e9c 100644 --- a/litellm-rust/crates/core/src/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/chat_completions/tests.rs @@ -4,8 +4,8 @@ use serde_json::{Map, Value, json}; use super::{ Error, prepare::{prepare_provider_request, resolve_request}, - types::{ChatCompletionsRequest, ProviderChatCompletionsRequest}, }; +use crate::chat_completions::types::{ChatCompletionsRequest, ProviderChatCompletionsRequest}; fn prepare_chat_completions_call( request: ChatCompletionsRequest<'_>, diff --git a/litellm-rust/crates/core/src/chat_completions/types.rs b/litellm-rust/crates/core/src/chat_completions/types.rs new file mode 100644 index 00000000000..1687ef0a792 --- /dev/null +++ b/litellm-rust/crates/core/src/chat_completions/types.rs @@ -0,0 +1,44 @@ +use std::time::Duration; + +use litellm_providers::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; +use litellm_types::llms::openai::ChatMessage; +use serde_json::{Map, Value}; + +/// A `/chat/completions` call as it crosses into the core. +/// +/// `optional_params` arrives already mapped to the provider's own parameter +/// names by the host, exactly as the messages route receives an already +/// Anthropic-shaped body. The core owns the conversation translation, the +/// provider call, and the response normalization. +pub struct ChatCompletionsRequest<'a> { + pub model: &'a str, + pub messages: Value, + pub optional_params: Map, + pub api_key: Option<&'a str>, + pub api_base: Option<&'a str>, + pub custom_llm_provider: Option<&'a str>, + pub extra_headers: Option>, + pub timeout: Option, +} + +pub struct ResolvedChatCompletionsRequest<'a> { + pub model: String, + pub config: &'static dyn BaseConfig, + pub messages: Vec, + pub optional_params: Map, + pub api_key: Option<&'a str>, + pub api_base: Option<&'a str>, + pub extra_headers: Option>, + pub timeout: Option, +} + +pub struct ProviderChatCompletionsRequest { + pub model: String, + pub config: &'static dyn BaseConfig, + pub url: String, + pub body: Value, + pub upstream_headers: Vec<(String, String)>, + pub auth: ChatCompletionsAuth, + pub optional_params: Map, + pub timeout: Option, +} diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/core/src/http_utils.rs index 060559322ea..a53ceee4382 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/core/src/http_utils.rs @@ -6,6 +6,7 @@ pub struct HeaderError { pub actual: &'static str, } +use litellm_core_utils::core_helpers::json_type_name; use serde_json::{Map, Value}; use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS; @@ -118,17 +119,6 @@ where as serde::Deserialize>::deserialize(deserializer).map(Some) } -pub fn json_type_name(value: &serde_json::Value) -> &'static str { - match value { - serde_json::Value::Null => "null", - serde_json::Value::Bool(_) => "bool", - serde_json::Value::Number(_) => "number", - serde_json::Value::String(_) => "string", - serde_json::Value::Array(_) => "array", - serde_json::Value::Object(_) => "object", - } -} - #[cfg(test)] mod tests { use serde_json::json; diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index b1474f3f6c4..8afb5783a34 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -1,19 +1,14 @@ pub mod audio_transcription; -pub mod call_arguments; pub mod chat_completions; pub mod constants; pub mod error; pub mod http_utils; -pub mod litellm_core_utils; pub mod llms; pub mod machine; mod media; pub mod messages; pub mod ocr; -pub mod params; pub mod responses; -mod serde_compat; pub mod transport; -mod url_utils; pub use error::Error; diff --git a/litellm-rust/crates/core/src/litellm_core_utils/mod.rs b/litellm-rust/crates/core/src/litellm_core_utils/mod.rs deleted file mode 100644 index 7e3b3e96dda..00000000000 --- a/litellm-rust/crates/core/src/litellm_core_utils/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod get_llm_provider_logic; diff --git a/litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs b/litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs index 2c540a4c436..86c6e010e84 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs +++ b/litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs @@ -1,19 +1,16 @@ use std::collections::HashMap; +use litellm_types::{ + llms::openai::{ChatCompletionThinkingBlock, ChatCompletionToolCallChunk}, + utils::{ChatCompletionChunk, ChatCompletionsUsage}, +}; use serde_json::Value; use super::super::experimental_pass_through::messages::streaming::{ AnthropicContentBlock, AnthropicContentBlockDelta, AnthropicMessagesStreamEvent, AnthropicStreamUsage, }; -use crate::chat_completions::{ - Error, - streaming::StreamTransformer, - types::{ - ChatCompletionChunk, ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, - ChatCompletionsUsage, - }, -}; +use crate::chat_completions::{Error, streaming::StreamTransformer}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum AnthropicJsonChunkType { diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs b/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs index 8a314bd3e56..e371a24c057 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs +++ b/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs @@ -1,10 +1,11 @@ use litellm_providers::anthropic::experimental_pass_through::messages::transformation::resolve_anthropic_api_base; +use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; use serde::{Deserialize, Serialize}; use serde_json::Value; use time::OffsetDateTime; use url::Url; -use crate::messages::{Error, types::AnthropicMessagesResponse}; +use crate::messages::Error; const BATCHES_PATH_SUFFIX: &str = "/v1/messages/batches"; diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs b/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs index 3e599f67eb3..82a2a54c961 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs +++ b/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs @@ -1,13 +1,8 @@ +use litellm_types::llms::anthropic_messages::anthropic_request::{AnthropicMessage, SystemPrompt}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::{ - constants::ANTHROPIC_OAUTH_TOKEN_PREFIX, - messages::{ - Error, - types::{AnthropicMessage, SystemPrompt}, - }, -}; +use crate::{constants::ANTHROPIC_OAUTH_TOKEN_PREFIX, messages::Error}; const COUNT_TOKENS_ENDPOINT: &str = "https://api.anthropic.com/v1/messages/count_tokens"; const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01"; @@ -97,10 +92,10 @@ impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation { #[cfg(test)] mod tests { + use litellm_types::llms::anthropic_messages::anthropic_request::MessageContent; use serde_json::{Map, json}; use super::*; - use crate::messages::types::MessageContent; fn message() -> AnthropicMessage { AnthropicMessage { diff --git a/litellm-rust/crates/core/src/llms/azure_ai/ocr/cohere_parse_transformation.rs b/litellm-rust/crates/core/src/llms/azure_ai/ocr/cohere_parse_transformation.rs index 71ea7a279a6..8698f062a94 100644 --- a/litellm-rust/crates/core/src/llms/azure_ai/ocr/cohere_parse_transformation.rs +++ b/litellm-rust/crates/core/src/llms/azure_ai/ocr/cohere_parse_transformation.rs @@ -1,7 +1,7 @@ +use litellm_core_utils::{call_arguments::CallArguments, url_utils::ApiUrl}; use serde_json::Value; use crate::{ - call_arguments::CallArguments, llms::{ base_llm::ocr::transformation::{BaseOcrConfig, OcrRequestContext}, cohere::ocr::{ @@ -15,7 +15,6 @@ use crate::{ document::{inline_remote_document, validate_inline_document}, types::{LiteLLMOcrResponse, OcrDocument, PreparedOcrRequest}, }, - url_utils::ApiUrl, }; #[derive(Default)] diff --git a/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs b/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs index 7ad4b4d120f..da6f7841709 100644 --- a/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs +++ b/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs @@ -3,6 +3,11 @@ use std::{collections::BTreeSet, time::Duration}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_auth::{InputSource, Sourced}; use litellm_auth_azure::AzureAuthInputs; +use litellm_core_utils::{ + call_arguments::CallArguments, + serde_compat::{FiniteF64, LaxI64}, + url_utils::ApiUrl, +}; use reqwest::Url; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::{Map, Value}; @@ -10,7 +15,6 @@ use serde_with::serde_as; use tokio::time::Instant; use crate::{ - call_arguments::CallArguments, constants::{ AZURE_DI_API_VERSION, AZURE_DI_DEFAULT_DPI, AZURE_DI_DEFAULT_HEIGHT, AZURE_DI_DEFAULT_WIDTH, AZURE_DI_SUBSCRIPTION_HEADER, OCR_POLL_RETRY_SECS, @@ -31,8 +35,6 @@ use crate::{ ResolvedOcrCredentials, }, }, - serde_compat::{FiniteF64, LaxI64}, - url_utils::ApiUrl, }; const AZURE_DI_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY"; diff --git a/litellm-rust/crates/core/src/llms/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/llms/azure_ai/ocr/transformation.rs index c6480ca6dac..7cd3973b97c 100644 --- a/litellm-rust/crates/core/src/llms/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/llms/azure_ai/ocr/transformation.rs @@ -1,17 +1,21 @@ use litellm_auth::{InputSource, Sourced}; use litellm_auth_azure::AzureAuthInputs; +use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl}; use serde_json::Value; -use crate::call_arguments::CallArguments; -use crate::constants::AZURE_AI_OCR_PATH; -use crate::llms::base_llm::ocr::transformation::{BaseOcrConfig, OcrRequestContext}; -use crate::llms::mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest}; -use crate::ocr::OcrClient; -use crate::ocr::document::{inline_remote_document, validate_inline_document}; -use crate::ocr::prepare::credential_env; -use crate::ocr::types::{LiteLLMOcrResponse, OcrConnection, OcrDocument, PreparedOcrRequest}; -use crate::params::OpaqueParams; -use crate::url_utils::ApiUrl; +use crate::{ + constants::AZURE_AI_OCR_PATH, + llms::{ + base_llm::ocr::transformation::{BaseOcrConfig, OcrRequestContext}, + mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest}, + }, + ocr::{ + OcrClient, + document::{inline_remote_document, validate_inline_document}, + prepare::credential_env, + types::{LiteLLMOcrResponse, OcrConnection, OcrDocument, PreparedOcrRequest}, + }, +}; const AZURE_AI_API_KEY_ENV: &str = "AZURE_AI_API_KEY"; const AZURE_AI_API_BASE_ENV: &str = "AZURE_AI_API_BASE"; @@ -306,9 +310,9 @@ mod tests { use serde_json::json; - use crate::ocr::LocalOcrHost; - use crate::ocr::test_support::{ - MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request, + use crate::ocr::{ + LocalOcrHost, + test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request}, }; #[tokio::test] @@ -388,16 +392,16 @@ mod tests { assert!(error.to_string().contains("data URI")); } - use std::sync::Arc; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; use litellm_auth::{ ResolvedCredential, SecretValue, TokenFuture, TokenProvider, TokenProviderHandle, }; - use crate::ocr::LiteLLMOcrRequest; - use crate::ocr::test_support::header; - use crate::ocr::wire::decode_request; + use crate::ocr::{LiteLLMOcrRequest, test_support::header, wire::decode_request}; #[derive(Debug)] struct CountingToken { diff --git a/litellm-rust/crates/core/src/llms/base_llm/ocr/transformation.rs b/litellm-rust/crates/core/src/llms/base_llm/ocr/transformation.rs index b9dca3c9bd4..ba54dfdaddd 100644 --- a/litellm-rust/crates/core/src/llms/base_llm/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/llms/base_llm/ocr/transformation.rs @@ -1,17 +1,15 @@ use std::future::Future; +use litellm_core_utils::call_arguments::CallArguments; use serde::{Serialize, de::DeserializeOwned}; use serde_json::Value; -use crate::{ - call_arguments::CallArguments, - ocr::{ - OcrClient, - route::OcrHost, - types::{ - LiteLLMOcrResponse, OcrConnection, OcrCredentialInputs, OcrDocument, OcrResponseFormat, - PreparedOcrRequest, ResolvedOcrCredentials, - }, +use crate::ocr::{ + OcrClient, + route::OcrHost, + types::{ + LiteLLMOcrResponse, OcrConnection, OcrCredentialInputs, OcrDocument, OcrResponseFormat, + PreparedOcrRequest, ResolvedOcrCredentials, }, }; diff --git a/litellm-rust/crates/core/src/llms/cohere/ocr/transformation.rs b/litellm-rust/crates/core/src/llms/cohere/ocr/transformation.rs index 573c0b833d8..adc27d22b70 100644 --- a/litellm-rust/crates/core/src/llms/cohere/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/llms/cohere/ocr/transformation.rs @@ -1,9 +1,13 @@ +use litellm_core_utils::{ + call_arguments::{CallArguments, parse_options}, + serde_compat::LaxI64, + url_utils::ApiUrl, +}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use serde_with::serde_as; use crate::{ - call_arguments::{CallArguments, parse_options}, constants::{COHERE_API_KEY_ENV, COHERE_PARSE_API_BASE}, llms::base_llm::ocr::transformation::{BaseOcrConfig, decode_and_normalize_response}, ocr::{ @@ -15,8 +19,6 @@ use crate::{ OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest, }, }, - serde_compat::LaxI64, - url_utils::ApiUrl, }; const COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4//8/AAX+Av4N70a4AAAAAElFTkSuQmCC"; diff --git a/litellm-rust/crates/core/src/llms/mistral/ocr/transformation.rs b/litellm-rust/crates/core/src/llms/mistral/ocr/transformation.rs index dac1ed7c68f..f565db15087 100644 --- a/litellm-rust/crates/core/src/llms/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/llms/mistral/ocr/transformation.rs @@ -1,8 +1,8 @@ +use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl}; use serde::{Deserialize, Serialize}; use serde_json::Value; use crate::{ - call_arguments::CallArguments, constants::MISTRAL_OCR_API_BASE, llms::base_llm::ocr::transformation::{BaseOcrConfig, decode_and_normalize_response}, ocr::{ @@ -13,8 +13,6 @@ use crate::{ OcrUsageInfo, PreparedOcrRequest, }, }, - params::OpaqueParams, - url_utils::ApiUrl, }; const MISTRAL_OCR_API_KEY_ENV_VAR: &str = "MISTRAL_API_KEY"; diff --git a/litellm-rust/crates/core/src/llms/openai/responses/transformation.rs b/litellm-rust/crates/core/src/llms/openai/responses/transformation.rs index 2c8916b6806..ddd160af7a4 100644 --- a/litellm-rust/crates/core/src/llms/openai/responses/transformation.rs +++ b/litellm-rust/crates/core/src/llms/openai/responses/transformation.rs @@ -1,6 +1,7 @@ +use litellm_types::responses::streaming_websocket::{ResponsesWsEvent, ResponsesWsTransformResult}; + use crate::responses::{ Error, - types::{ResponsesWsEvent, ResponsesWsTransformResult}, websocket::{ResponsesWebSocketProviderConfig, enforce_model}, }; diff --git a/litellm-rust/crates/core/src/llms/reducto/ocr/transformation.rs b/litellm-rust/crates/core/src/llms/reducto/ocr/transformation.rs index 4c5323ef50e..2077b46e2d3 100644 --- a/litellm-rust/crates/core/src/llms/reducto/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/llms/reducto/ocr/transformation.rs @@ -1,10 +1,14 @@ use std::collections::BTreeMap; +use litellm_core_utils::{ + call_arguments::{CallArguments, compose_body}, + params::OpaqueParams, + url_utils::ApiUrl, +}; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::{Map, Value, json}; use crate::{ - call_arguments::{CallArguments, compose_body}, constants::{REDUCTO_API_BASE, REDUCTO_API_KEY_ENV, REDUCTO_ID_PREFIX}, llms::base_llm::ocr::transformation::{ BaseOcrConfig, OcrRequestContext, decode_and_normalize_response, @@ -18,8 +22,6 @@ use crate::{ OcrUsageInfo, PreparedOcrRequest, }, }, - params::OpaqueParams, - url_utils::ApiUrl, }; #[derive(Clone, Debug, Serialize, Deserialize)] @@ -70,9 +72,9 @@ struct ReductoResult { #[serde_with::serde_as] #[derive(Clone, Debug, Default, Deserialize)] struct ReductoUsage { - #[serde_as(deserialize_as = "Option")] + #[serde_as(deserialize_as = "Option")] pub num_pages: Option, - #[serde_as(deserialize_as = "Option")] + #[serde_as(deserialize_as = "Option")] pub credits: Option, } @@ -624,7 +626,10 @@ mod tests { }) ); let absent = ReductoParseV3Config - .map_ocr_params(&crate::call_arguments::CallArguments::default(), "parse-v3") + .map_ocr_params( + &litellm_core_utils::call_arguments::CallArguments::default(), + "parse-v3", + ) .unwrap(); assert_eq!(serde_json::to_value(absent).unwrap(), json!({})); } diff --git a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/deepseek_transformation.rs b/litellm-rust/crates/core/src/llms/vertex_ai/ocr/deepseek_transformation.rs index 6aece071d26..c53adb5befc 100644 --- a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/deepseek_transformation.rs +++ b/litellm-rust/crates/core/src/llms/vertex_ai/ocr/deepseek_transformation.rs @@ -1,10 +1,10 @@ use litellm_auth_gcp::{self as vertex, VertexConfig}; +use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use super::transformation::VertexAiOcrConfig; use crate::{ - call_arguments::CallArguments, llms::base_llm::ocr::transformation::{BaseOcrConfig, OcrRequestContext}, ocr::{ OcrClient, @@ -14,8 +14,6 @@ use crate::{ OcrUsageInfo, PreparedOcrRequest, }, }, - params::OpaqueParams, - url_utils::ApiUrl, }; const DEFAULT_API_BASE: &str = "https://aiplatform.googleapis.com"; @@ -82,7 +80,7 @@ enum DeepSeekContent { #[derive(Deserialize)] struct DeepSeekPage { #[serde(default)] - #[serde_as(deserialize_as = "crate::serde_compat::LaxI64")] + #[serde_as(deserialize_as = "litellm_core_utils::serde_compat::LaxI64")] index: i64, #[serde(default)] markdown: String, @@ -434,8 +432,12 @@ mod tests { json!({}) ); assert_eq!( - crate::call_arguments::compose_body(&arguments, &json!({"model":"deepseek-ocr"}), &[]) - .unwrap(), + litellm_core_utils::call_arguments::compose_body( + &arguments, + &json!({"model":"deepseek-ocr"}), + &[] + ) + .unwrap(), json!({"model":"deepseek-ocr","temperature":0.5,"extension":null}) ); } diff --git a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/llms/vertex_ai/ocr/transformation.rs index bd7c5da7632..53e2d2291b0 100644 --- a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/llms/vertex_ai/ocr/transformation.rs @@ -1,9 +1,9 @@ use litellm_auth_gcp::{self as vertex, VertexConfig}; +use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl}; use serde_json::Value; use super::common_utils::validate_destination; use crate::{ - call_arguments::CallArguments, llms::{ base_llm::ocr::transformation::{BaseOcrConfig, OcrEnvironment, OcrRequestContext}, mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest}, @@ -14,8 +14,6 @@ use crate::{ prepare::credential_env, types::{LiteLLMOcrResponse, OcrConnection, OcrDocument, PreparedOcrRequest}, }, - params::OpaqueParams, - url_utils::ApiUrl, }; const DEFAULT_LOCATION: &str = "us-central1"; diff --git a/litellm-rust/crates/core/src/messages/error.rs b/litellm-rust/crates/core/src/messages/error.rs index cdb4de4645f..7a9c1c1d20a 100644 --- a/litellm-rust/crates/core/src/messages/error.rs +++ b/litellm-rust/crates/core/src/messages/error.rs @@ -1,3 +1,5 @@ +use litellm_providers::base_llm::chat::transformation::Error as LlmError; + #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { #[error("invalid provider: {0}")] @@ -28,18 +30,15 @@ pub enum Error { InvalidBedrockBase64(String), } -impl From for Error { - fn from(error: litellm_providers::messages::Error) -> Self { +impl From for Error { + fn from(error: LlmError) -> Self { match error { - litellm_providers::messages::Error::MissingField(field) => Self::MissingField(field), - litellm_providers::messages::Error::InvalidRequest(message) => { - Self::InvalidRequest(message) - } - litellm_providers::messages::Error::InvalidResponse(message) => { - Self::InvalidResponse(message) - } - litellm_providers::messages::Error::Unsupported(reason) => Self::Unsupported(reason), - litellm_providers::messages::Error::Auth(error) => Self::Auth(error), + error @ LlmError::InvalidType { .. } => Self::InvalidRequest(error.to_string()), + LlmError::MissingField(field) => Self::MissingField(field), + LlmError::InvalidRequest(message) => Self::InvalidRequest(message), + LlmError::InvalidResponse(message) => Self::InvalidResponse(message), + LlmError::Unsupported(reason) => Self::Unsupported(reason), + LlmError::Auth(error) => Self::Auth(error), } } } diff --git a/litellm-rust/crates/core/src/messages/handler.rs b/litellm-rust/crates/core/src/messages/handler.rs index ff3ae5765ff..b006d301587 100644 --- a/litellm-rust/crates/core/src/messages/handler.rs +++ b/litellm-rust/crates/core/src/messages/handler.rs @@ -1,11 +1,13 @@ +use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; + use super::{ - Error, - client::http_client, - common_utils::truncate_error_body, + Error, client::http_client, common_utils::truncate_error_body, prepare::prepare_provider_request, - types::{AnthropicMessagesResponse, MessagesRequest}, }; -use crate::{constants::ANTHROPIC_MESSAGES_PROVIDER, http_utils::http_request}; +use crate::{ + constants::ANTHROPIC_MESSAGES_PROVIDER, http_utils::http_request, + messages::types::MessagesRequest, +}; pub(super) async fn execute_messages_provider_call( request: MessagesRequest<'_>, diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index 812094f637c..c3d7bea48ff 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -8,14 +8,16 @@ //! can splice the event stream to its own caller. mod error; +pub mod types; pub use error::Error; mod client; mod common_utils; mod handler; mod prepare; use handler::{execute_messages_provider_call, execute_messages_provider_stream}; -pub use litellm_providers::messages::types; -use types::{AnthropicMessagesResponse, MessagesRequest}; +use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; + +use crate::messages::types::MessagesRequest; pub async fn messages(request: MessagesRequest<'_>) -> Result { execute_messages_provider_call(request).await diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index 4a6c871172f..d7e996d74e8 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,3 +1,4 @@ +use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; use litellm_providers::base_llm::anthropic_messages::transformation::{ BaseAnthropicMessagesConfig, MessagesAuthStrategy, }; @@ -6,11 +7,8 @@ use serde_json::{Map, Value}; use super::{ Error, common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers}, - types::{MessagesRequest, ProviderMessagesRequest}, -}; -use crate::litellm_core_utils::get_llm_provider_logic::{ - CustomLlmProvider, get_custom_llm_provider, }; +use crate::messages::types::{MessagesRequest, ProviderMessagesRequest}; pub(super) fn prepare_provider_request( request: MessagesRequest<'_>, diff --git a/litellm-rust/crates/core/src/messages/tests.rs b/litellm-rust/crates/core/src/messages/tests.rs index 98b9bd626a9..4b9c5ca2aeb 100644 --- a/litellm-rust/crates/core/src/messages/tests.rs +++ b/litellm-rust/crates/core/src/messages/tests.rs @@ -12,8 +12,8 @@ use super::{ has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body, }, messages, - types::MessagesRequest, }; +use crate::messages::types::MessagesRequest; async fn read_http_request(socket: &mut TcpStream) -> String { let mut request = Vec::new(); diff --git a/litellm-rust/crates/core/src/messages/types.rs b/litellm-rust/crates/core/src/messages/types.rs new file mode 100644 index 00000000000..928eca222be --- /dev/null +++ b/litellm-rust/crates/core/src/messages/types.rs @@ -0,0 +1,24 @@ +use std::time::Duration; + +use litellm_providers::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig; +use serde_json::{Map, Value}; + +pub struct MessagesRequest<'a> { + pub model: &'a str, + pub body: Value, + pub api_key: Option<&'a str>, + pub api_base: Option<&'a str>, + pub custom_llm_provider: Option<&'a str>, + pub extra_headers: Option>, + pub timeout: Option, +} + +pub struct ProviderMessagesRequest { + pub provider: String, + pub model: String, + pub config: &'static dyn BaseAnthropicMessagesConfig, + pub url: String, + pub body: Value, + pub upstream_headers: Vec<(String, String)>, + pub timeout: Option, +} diff --git a/litellm-rust/crates/core/src/ocr/arguments.rs b/litellm-rust/crates/core/src/ocr/arguments.rs index 2b27496fb5f..79d24eb83b2 100644 --- a/litellm-rust/crates/core/src/ocr/arguments.rs +++ b/litellm-rust/crates/core/src/ocr/arguments.rs @@ -1,5 +1,6 @@ +use litellm_core_utils::call_arguments::ArgumentSpec; + use super::provider_config::{OcrConfigKind, resolve_provider_config}; -use crate::call_arguments::ArgumentSpec; const COMMON_OPTION_FIELDS: &[&str] = &["req_format", "extra_body", "max_response_bytes"]; const AZURE_AUTH_OPTION_FIELDS: &[&str] = &[ diff --git a/litellm-rust/crates/core/src/ocr/error.rs b/litellm-rust/crates/core/src/ocr/error.rs index 4906b5515b9..8d8372170cc 100644 --- a/litellm-rust/crates/core/src/ocr/error.rs +++ b/litellm-rust/crates/core/src/ocr/error.rs @@ -97,13 +97,13 @@ pub enum Error { #[error(transparent)] Transport(#[from] crate::transport::Error), #[error(transparent)] - Params(#[from] crate::params::Error), + Params(#[from] litellm_core_utils::params::Error), #[error(transparent)] Headers(#[from] crate::http_utils::HeaderError), } -impl From for Error { - fn from(error: crate::call_arguments::ArgumentError) -> Self { +impl From for Error { + fn from(error: litellm_core_utils::call_arguments::ArgumentError) -> Self { Self::RequestField { path: format!("optional_params.{}", error.path), } diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index 2de72660794..f65e3f1b644 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -2,9 +2,11 @@ use litellm_callbacks::event::{Passthrough, RequestContext, WireRequest}; use serde::Serialize; use serde_json::{Map, Value}; -use super::OcrClient; -use super::route::OcrHost; -use super::types::{OcrConnection, OcrDocument, PreparedOcrRequest, ResolvedOcrRequest}; +use super::{ + OcrClient, + route::OcrHost, + types::{OcrConnection, OcrDocument, PreparedOcrRequest, ResolvedOcrRequest}, +}; pub(crate) async fn transform_request_body( client: &OcrClient, @@ -17,7 +19,7 @@ pub(crate) async fn transform_request_body( where B: Serialize, { - let composed = crate::call_arguments::compose_body( + let composed = litellm_core_utils::call_arguments::compose_body( &request.optional_params, &body, request.config.get_supported_ocr_params(&request.model), @@ -191,10 +193,9 @@ pub(crate) fn prepare_request_for_test(request: ResolvedOcrRequest) -> PreparedO #[cfg(test)] mod tests { + use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options}; use serde_json::json; - use crate::call_arguments::{CallArguments, compose_body, parse_options}; - #[derive(serde::Deserialize)] struct KnownParams { pages: Option>, diff --git a/litellm-rust/crates/core/src/ocr/provider_config.rs b/litellm-rust/crates/core/src/ocr/provider_config.rs index 0121f2dfdf4..2abc7825405 100644 --- a/litellm-rust/crates/core/src/ocr/provider_config.rs +++ b/litellm-rust/crates/core/src/ocr/provider_config.rs @@ -1,3 +1,4 @@ +use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; use strum::{EnumString, IntoStaticStr}; use super::{ @@ -7,21 +8,18 @@ use super::{ ResolvedOcrCredentials, }, }; -use crate::{ - litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}, - llms::{ - azure_ai::ocr::{ - cohere_parse_transformation::AzureAICohereParseConfig, - document_intelligence::transformation::AzureDocumentIntelligenceOcrConfig, - transformation::AzureAiOcrConfig, - }, - base_llm::ocr::transformation::{BaseOcrConfig, OcrResponseContext}, - cohere::ocr::transformation::CohereParseConfig, - mistral::ocr::transformation::MistralOcrConfig, - reducto::ocr::transformation::{ReductoParseLegacyConfig, ReductoParseV3Config}, - vertex_ai::ocr::{ - deepseek_transformation::VertexAIDeepSeekOCRConfig, transformation::VertexAiOcrConfig, - }, +use crate::llms::{ + azure_ai::ocr::{ + cohere_parse_transformation::AzureAICohereParseConfig, + document_intelligence::transformation::AzureDocumentIntelligenceOcrConfig, + transformation::AzureAiOcrConfig, + }, + base_llm::ocr::transformation::{BaseOcrConfig, OcrResponseContext}, + cohere::ocr::transformation::CohereParseConfig, + mistral::ocr::transformation::MistralOcrConfig, + reducto::ocr::transformation::{ReductoParseLegacyConfig, ReductoParseV3Config}, + vertex_ai::ocr::{ + deepseek_transformation::VertexAIDeepSeekOCRConfig, transformation::VertexAiOcrConfig, }, }; diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs index 91851540c26..40bd4c7491b 100644 --- a/litellm-rust/crates/core/src/ocr/types.rs +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -2,16 +2,16 @@ use std::{collections::BTreeMap, path::PathBuf, time::Duration}; use bytes::Bytes; use litellm_auth::{InputSource, Sourced, TokenProviderHandle}; +use litellm_core_utils::{ + call_arguments::CallArguments, + serde_compat::{FiniteF64, LaxI64}, +}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use serde_with::serde_as; use super::provider_config::{OcrConfigKind, resolve_provider_config}; -use crate::{ - call_arguments::CallArguments, - constants::OCR_HTTP_TIMEOUT_SECS, - serde_compat::{FiniteF64, LaxI64}, -}; +use crate::constants::OCR_HTTP_TIMEOUT_SECS; #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(tag = "type")] diff --git a/litellm-rust/crates/core/src/ocr/wire.rs b/litellm-rust/crates/core/src/ocr/wire.rs index 603e455ace1..a4e311bb786 100644 --- a/litellm-rust/crates/core/src/ocr/wire.rs +++ b/litellm-rust/crates/core/src/ocr/wire.rs @@ -10,11 +10,11 @@ use super::{Error, LiteLLMOcrRequest, OcrConnectionInputs, OcrDocument, OcrDocum pub fn consumed_optional_params( model: &str, provider: Option<&str>, -) -> Result, Error> { +) -> Result, Error> { let specs = super::consumed_optional_params(model, provider)?; Ok(consumed_optional_param_names(model, provider)? .into_iter() - .map(|name| crate::call_arguments::ArgumentSpec { + .map(|name| litellm_core_utils::call_arguments::ArgumentSpec { name, secret: specs.iter().any(|spec| spec.name == name && spec.secret), }) diff --git a/litellm-rust/crates/core/src/responses/mod.rs b/litellm-rust/crates/core/src/responses/mod.rs index 6af2bf0c199..bc0f71896e5 100644 --- a/litellm-rust/crates/core/src/responses/mod.rs +++ b/litellm-rust/crates/core/src/responses/mod.rs @@ -1,4 +1,3 @@ mod error; pub use error::Error; -pub mod types; pub mod websocket; diff --git a/litellm-rust/crates/core/src/responses/websocket.rs b/litellm-rust/crates/core/src/responses/websocket.rs index 7758cb2414c..a91fbcf272f 100644 --- a/litellm-rust/crates/core/src/responses/websocket.rs +++ b/litellm-rust/crates/core/src/responses/websocket.rs @@ -6,6 +6,9 @@ use std::{ }; use futures_util::{SinkExt, StreamExt}; +use litellm_types::responses::streaming_websocket::{ + ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult, +}; use rustls::{ClientConfig, RootCertStore}; use tokio::{net::TcpStream, sync::Mutex}; use tokio_tungstenite::{ @@ -20,10 +23,7 @@ use tokio_tungstenite::{ }; use super::Error; -use crate::{ - constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH}, - responses::types::{ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult}, -}; +use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH}; pub trait ResponsesWebSocketProviderConfig: Sync { fn supports_native_websocket(&self) -> bool { diff --git a/litellm-rust/crates/core/tests/ocr/passthrough.rs b/litellm-rust/crates/core/tests/ocr/passthrough.rs index c1cd1adf291..a0fbfeba0c2 100644 --- a/litellm-rust/crates/core/tests/ocr/passthrough.rs +++ b/litellm-rust/crates/core/tests/ocr/passthrough.rs @@ -1,15 +1,19 @@ -use std::collections::BTreeSet; -use std::sync::{Arc, Mutex}; +use std::{ + collections::BTreeSet, + sync::{Arc, Mutex}, +}; use litellm_callbacks::event::{RequestContext, WireRequest}; use rstest::rstest; use rstest_reuse::{self, apply, template}; use serde_json::{Map, Value, json}; -use super::LocalOcrHost; -use super::test_support::{ - MockResponse, SERVED_DOCUMENT, document_server, mock_server, perform_ocr_with, request_body, - wire_request_with_document, +use super::{ + LocalOcrHost, + test_support::{ + MockResponse, SERVED_DOCUMENT, document_server, mock_server, perform_ocr_with, + request_body, wire_request_with_document, + }, }; #[derive(Clone, Copy, Debug)] diff --git a/litellm-rust/crates/providers/Cargo.toml b/litellm-rust/crates/providers/Cargo.toml index e1c8f2c50d4..f137e2ddbe4 100644 --- a/litellm-rust/crates/providers/Cargo.toml +++ b/litellm-rust/crates/providers/Cargo.toml @@ -6,6 +6,8 @@ license.workspace = true repository.workspace = true [dependencies] +litellm-types.workspace = true +litellm-core-utils.workspace = true litellm-auth.workspace = true litellm-auth-aws.workspace = true serde.workspace = true diff --git a/litellm-rust/crates/providers/src/anthropic/chat/tests.rs b/litellm-rust/crates/providers/src/anthropic/chat/tests.rs index 18b6efb13fd..40e89c52c3c 100644 --- a/litellm-rust/crates/providers/src/anthropic/chat/tests.rs +++ b/litellm-rust/crates/providers/src/anthropic/chat/tests.rs @@ -1,7 +1,7 @@ use serde_json::json; use super::*; -use crate::chat::Error; +use crate::base_llm::chat::transformation::Error; fn messages(value: Value) -> Vec { serde_json::from_value(value).expect("valid messages") diff --git a/litellm-rust/crates/providers/src/anthropic/chat/transformation.rs b/litellm-rust/crates/providers/src/anthropic/chat/transformation.rs index 5288eebbb2f..21fa4e9f82e 100644 --- a/litellm-rust/crates/providers/src/anthropic/chat/transformation.rs +++ b/litellm-rust/crates/providers/src/anthropic/chat/transformation.rs @@ -1,18 +1,24 @@ +use litellm_core_utils::{ + core_helpers::{finish_reason_for, unix_now, usage_from_parts}, + prompt_templates::factory::{Conversation, build_conversation}, +}; +use litellm_types::{ + llms::openai::ChatMessage, + utils::{ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse}, +}; use serde_json::{Map, Value, json}; -use crate::anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX; -use crate::anthropic::experimental_pass_through::messages::transformation::{ - complete_anthropic_url, resolve_anthropic_api_key, -}; -use crate::base_llm::chat::transformation::{ - BaseConfig, ChatCompletionsAuth, Unsupported, unsupported_message, unsupported_param, -}; -use crate::chat::Error; -use crate::chat::conversation::{Conversation, build_conversation}; -use crate::chat::response_utils::{finish_reason_for, unix_now, usage_from_parts}; -use crate::chat::types::{ - ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, ChatMessage, - ProviderChatRequestData, ProviderChatResponseData, +use crate::{ + anthropic::{ + ANTHROPIC_OAUTH_TOKEN_PREFIX, + experimental_pass_through::messages::transformation::{ + complete_anthropic_url, resolve_anthropic_api_key, + }, + }, + base_llm::chat::transformation::{ + BaseConfig, ChatCompletionsAuth, Error, ProviderChatRequestData, ProviderChatResponseData, + Unsupported, unsupported_message, unsupported_param, + }, }; /// Anthropic parameter names, post `map_openai_params`, that the Rust path can diff --git a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/transformation.rs b/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/transformation.rs index beabe440269..c791749ac6d 100644 --- a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/transformation.rs +++ b/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/transformation.rs @@ -1,5 +1,6 @@ -use crate::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig; -use crate::messages::Error; +use crate::base_llm::{ + anthropic_messages::transformation::BaseAnthropicMessagesConfig, chat::transformation::Error, +}; const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; const ANTHROPIC_API_BASE_ENV: &str = "ANTHROPIC_API_BASE"; diff --git a/litellm-rust/crates/providers/src/audio_transcription/mod.rs b/litellm-rust/crates/providers/src/audio_transcription/mod.rs deleted file mode 100644 index 278b049e8f9..00000000000 --- a/litellm-rust/crates/providers/src/audio_transcription/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -use thiserror::Error; - -#[derive(Clone, Debug, PartialEq, Eq, Error)] -pub enum Error { - #[error("expected {expected}, got {actual}")] - InvalidType { - expected: &'static str, - actual: &'static str, - }, - #[error("missing required field: {0}")] - MissingField(&'static str), - #[error("invalid request: {0}")] - InvalidRequest(String), - #[error("invalid response: {0}")] - InvalidResponse(String), - #[error(transparent)] - Auth(#[from] litellm_auth::Error), -} - -pub fn json_type_name(value: &serde_json::Value) -> &'static str { - match value { - serde_json::Value::Null => "null", - serde_json::Value::Bool(_) => "boolean", - serde_json::Value::Number(_) => "number", - serde_json::Value::String(_) => "string", - serde_json::Value::Array(_) => "array", - serde_json::Value::Object(_) => "object", - } -} - -pub mod types; diff --git a/litellm-rust/crates/providers/src/azure_ai/anthropic/messages_transformation.rs b/litellm-rust/crates/providers/src/azure_ai/anthropic/messages_transformation.rs index a79b9038144..99f55f18afc 100644 --- a/litellm-rust/crates/providers/src/azure_ai/anthropic/messages_transformation.rs +++ b/litellm-rust/crates/providers/src/azure_ai/anthropic/messages_transformation.rs @@ -1,15 +1,19 @@ +use litellm_types::llms::anthropic_messages::{ + anthropic_request::{ + AnthropicMessage, AnthropicMessagesRequest, ContentBlock, MessageContent, SystemPrompt, + }, + anthropic_response::AnthropicMessagesResponse, +}; use serde_json::{Map, Value}; -use crate::anthropic::experimental_pass_through::messages::transformation::{ - ANTHROPIC_MESSAGES_CONFIG, AnthropicMessagesConfig, non_empty, -}; -use crate::base_llm::anthropic_messages::transformation::{ - BaseAnthropicMessagesConfig, MessagesAuthStrategy, -}; -use crate::messages::Error; -use crate::messages::types::{ - AnthropicMessage, AnthropicMessagesRequest, AnthropicMessagesResponse, ContentBlock, - MessageContent, SystemPrompt, +use crate::{ + anthropic::experimental_pass_through::messages::transformation::{ + ANTHROPIC_MESSAGES_CONFIG, AnthropicMessagesConfig, non_empty, + }, + base_llm::{ + anthropic_messages::transformation::{BaseAnthropicMessagesConfig, MessagesAuthStrategy}, + chat::transformation::Error, + }, }; const AZURE_API_KEY_ENV: &str = "AZURE_API_KEY"; diff --git a/litellm-rust/crates/providers/src/base_llm/anthropic_messages/transformation.rs b/litellm-rust/crates/providers/src/base_llm/anthropic_messages/transformation.rs index 37bf8884ec0..5b4afb601d2 100644 --- a/litellm-rust/crates/providers/src/base_llm/anthropic_messages/transformation.rs +++ b/litellm-rust/crates/providers/src/base_llm/anthropic_messages/transformation.rs @@ -1,5 +1,8 @@ -use crate::messages::Error; -use crate::messages::types::{AnthropicMessagesRequest, AnthropicMessagesResponse}; +use litellm_types::llms::anthropic_messages::{ + anthropic_request::AnthropicMessagesRequest, anthropic_response::AnthropicMessagesResponse, +}; + +use crate::base_llm::chat::transformation::Error; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MessagesAuthStrategy { diff --git a/litellm-rust/crates/providers/src/base_llm/audio_transcription/transformation.rs b/litellm-rust/crates/providers/src/base_llm/audio_transcription/transformation.rs index b478bd4caab..dd4588732be 100644 --- a/litellm-rust/crates/providers/src/base_llm/audio_transcription/transformation.rs +++ b/litellm-rust/crates/providers/src/base_llm/audio_transcription/transformation.rs @@ -1,9 +1,25 @@ +use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use crate::audio_transcription::Error; -use crate::audio_transcription::types::{ - AudioTranscriptionRequestData, AudioTranscriptionResponseData, -}; +use crate::base_llm::chat::transformation::Error; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct AudioTranscriptionRequestData { + pub body: Value, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct AudioTranscriptionResponseData { + pub text: String, +} + +impl AudioTranscriptionResponseData { + pub fn into_json(self) -> Value { + serde_json::json!({ + "text": self.text, + }) + } +} #[derive(Clone, Debug, PartialEq, Eq)] pub enum AudioTranscriptionAuth { diff --git a/litellm-rust/crates/providers/src/base_llm/chat/transformation.rs b/litellm-rust/crates/providers/src/base_llm/chat/transformation.rs index 5d81dc1a85e..ac0450c25f0 100644 --- a/litellm-rust/crates/providers/src/base_llm/chat/transformation.rs +++ b/litellm-rust/crates/providers/src/base_llm/chat/transformation.rs @@ -1,10 +1,39 @@ +use litellm_types::{ + llms::openai::{ChatMessage, ChatMessageContent}, + utils::ChatCompletionsResponse, +}; use serde_json::{Map, Value}; -use crate::chat::Error; -use crate::chat::types::{ - ChatCompletionsResponse, ChatMessage, ChatMessageContent, ProviderChatRequestData, - ProviderChatResponseData, -}; +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("expected {expected}, got {actual}")] + InvalidType { + expected: &'static str, + actual: &'static str, + }, + #[error("missing required field: {0}")] + MissingField(&'static str), + #[error("invalid request: {0}")] + InvalidRequest(String), + #[error("invalid response: {0}")] + InvalidResponse(String), + #[error("unsupported: {0}")] + Unsupported(&'static str), + #[error(transparent)] + Auth(#[from] litellm_auth::Error), +} + +/// The provider-shaped request body a config produces. Named rather than a bare +/// `Value` so the transform contract stays a typed one, mirroring +/// [`crate::base_llm::audio_transcription::transformation::AudioTranscriptionRequestData`]. +pub struct ProviderChatRequestData { + pub body: Value, +} + +/// The raw provider response body handed back to a config for normalization. +pub struct ProviderChatResponseData { + pub body: Value, +} pub const STREAM_PARAM: &str = "stream"; diff --git a/litellm-rust/crates/providers/src/bedrock/audio_transcription/mod.rs b/litellm-rust/crates/providers/src/bedrock/audio_transcription/mod.rs index 7da2aa42a51..39734d844da 100644 --- a/litellm-rust/crates/providers/src/bedrock/audio_transcription/mod.rs +++ b/litellm-rust/crates/providers/src/bedrock/audio_transcription/mod.rs @@ -1,15 +1,18 @@ +use litellm_auth_aws::{ + bedrock_model_id_and_region, + constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}, + resolve_bedrock_region, +}; +use litellm_core_utils::core_helpers::json_type_name; use serde_json::{Map, Value, json}; -use crate::audio_transcription::Error; -use crate::audio_transcription::json_type_name; -use crate::audio_transcription::types::{ - AudioTranscriptionRequestData, AudioTranscriptionResponseData, +use crate::base_llm::{ + audio_transcription::transformation::{ + AudioTranscriptionAuth, AudioTranscriptionRequestData, AudioTranscriptionResponseData, + BaseAudioTranscriptionConfig, + }, + chat::transformation::Error, }; -use crate::base_llm::audio_transcription::transformation::{ - AudioTranscriptionAuth, BaseAudioTranscriptionConfig, -}; -use litellm_auth_aws::constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}; -use litellm_auth_aws::{bedrock_model_id_and_region, resolve_bedrock_region}; const SUPPORTED_PARAMS: &[&str] = &["language", "prompt", "temperature", "response_format"]; diff --git a/litellm-rust/crates/providers/src/bedrock/chat/converse_transformation.rs b/litellm-rust/crates/providers/src/bedrock/chat/converse_transformation.rs index 85ba3be9b07..23c6c5c61bd 100644 --- a/litellm-rust/crates/providers/src/bedrock/chat/converse_transformation.rs +++ b/litellm-rust/crates/providers/src/bedrock/chat/converse_transformation.rs @@ -1,18 +1,25 @@ +use litellm_auth_aws::{ + bedrock_model_id_and_region, + constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE}, + resolve_bedrock_region, +}; +use litellm_core_utils::{ + core_helpers::{finish_reason_for, unix_now, usage_from_parts}, + prompt_templates::factory::{Conversation, TurnRole, build_conversation}, +}; +use litellm_types::{ + llms::openai::{ChatMessage, ChatMessageContent}, + utils::{ + ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, + ChatCompletionsUsage, + }, +}; use serde_json::{Map, Value, json}; use crate::base_llm::chat::transformation::{ - BaseConfig, ChatCompletionsAuth, Unsupported, unsupported_message, unsupported_param, + BaseConfig, ChatCompletionsAuth, Error, ProviderChatRequestData, ProviderChatResponseData, + Unsupported, unsupported_message, unsupported_param, }; -use crate::chat::Error; -use crate::chat::conversation::{Conversation, TurnRole, build_conversation}; -use crate::chat::response_utils::{finish_reason_for, unix_now, usage_from_parts}; -use crate::chat::types::{ - ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, - ChatCompletionsUsage, ChatMessage, ChatMessageContent, ProviderChatRequestData, - ProviderChatResponseData, -}; -use litellm_auth_aws::constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE}; -use litellm_auth_aws::{bedrock_model_id_and_region, resolve_bedrock_region}; /// Converse parameter names, post `map_openai_params`, that the Rust path can /// place verbatim in `inferenceConfig`. diff --git a/litellm-rust/crates/providers/src/bedrock/chat/tests.rs b/litellm-rust/crates/providers/src/bedrock/chat/tests.rs index cfa0c902096..cca5cbda41a 100644 --- a/litellm-rust/crates/providers/src/bedrock/chat/tests.rs +++ b/litellm-rust/crates/providers/src/bedrock/chat/tests.rs @@ -1,7 +1,7 @@ use serde_json::json; use super::*; -use crate::chat::Error; +use crate::base_llm::chat::transformation::Error; fn messages(value: Value) -> Vec { serde_json::from_value(value).expect("valid messages") diff --git a/litellm-rust/crates/providers/src/chat/mod.rs b/litellm-rust/crates/providers/src/chat/mod.rs deleted file mode 100644 index 93892657c75..00000000000 --- a/litellm-rust/crates/providers/src/chat/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -use thiserror::Error; - -pub const EMPTY_TEXT_PLACEHOLDER: &str = " "; - -#[derive(Clone, Debug, PartialEq, Eq, Error)] -pub enum Error { - #[error("missing required field: {0}")] - MissingField(&'static str), - #[error("invalid request: {0}")] - InvalidRequest(String), - #[error("invalid response: {0}")] - InvalidResponse(String), - #[error("unsupported: {0}")] - Unsupported(&'static str), - #[error(transparent)] - Auth(#[from] litellm_auth::Error), -} - -pub mod conversation; -pub mod response_utils; -pub mod types; diff --git a/litellm-rust/crates/providers/src/chat/types.rs b/litellm-rust/crates/providers/src/chat/types.rs deleted file mode 100644 index d61892624cf..00000000000 --- a/litellm-rust/crates/providers/src/chat/types.rs +++ /dev/null @@ -1,202 +0,0 @@ -use std::time::Duration; - -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; - -use crate::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; - -/// A `/chat/completions` call as it crosses into the core. -/// -/// `optional_params` arrives already mapped to the provider's own parameter -/// names by the host, exactly as the messages route receives an already -/// Anthropic-shaped body. The core owns the conversation translation, the -/// provider call, and the response normalization. -pub struct ChatCompletionsRequest<'a> { - pub model: &'a str, - pub messages: Value, - pub optional_params: Map, - pub api_key: Option<&'a str>, - pub api_base: Option<&'a str>, - pub custom_llm_provider: Option<&'a str>, - pub extra_headers: Option>, - pub timeout: Option, -} - -pub struct ResolvedChatCompletionsRequest<'a> { - pub model: String, - pub config: &'static dyn BaseConfig, - pub messages: Vec, - pub optional_params: Map, - pub api_key: Option<&'a str>, - pub api_base: Option<&'a str>, - pub extra_headers: Option>, - pub timeout: Option, -} - -pub struct ProviderChatCompletionsRequest { - pub model: String, - pub config: &'static dyn BaseConfig, - pub url: String, - pub body: Value, - pub upstream_headers: Vec<(String, String)>, - pub auth: ChatCompletionsAuth, - pub optional_params: Map, - pub timeout: Option, -} - -/// The provider-shaped request body a config produces. Named rather than a bare -/// `Value` so the transform contract stays a typed one, mirroring -/// [`crate::audio_transcription::types::AudioTranscriptionRequestData`]. -pub struct ProviderChatRequestData { - pub body: Value, -} - -/// The raw provider response body handed back to a config for normalization. -pub struct ProviderChatResponseData { - pub body: Value, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ChatMessageContent { - Text(String), - Parts(Vec), -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatMessage { - pub role: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub content: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, - #[serde(flatten)] - pub extra: Map, -} - -/// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python -/// path reports so cost tracking sees the same numbers on either path. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct PromptTokensDetails { - pub cached_tokens: u64, - pub cache_creation_tokens: u64, - pub text_tokens: u64, -} - -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionsUsage { - pub prompt_tokens: u64, - pub completion_tokens: u64, - pub total_tokens: u64, - pub prompt_tokens_details: PromptTokensDetails, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionsChoiceMessage { - pub role: String, - // Whether an empty turn is `None` or `""` is the provider's choice, not a - // shared invariant: Anthropic's transform ends on `merged_text or None` - // while Converse assigns the joined string unconditionally. Each config - // mirrors its own, so keep this optional and serialize it even when None. - pub content: Option, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionsChoice { - pub index: u64, - pub message: ChatCompletionsChoiceMessage, - pub finish_reason: String, -} - -/// The normalized response handed back to the host. -/// -/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the -/// `ModelResponse` it already created, and echoing the provider's own id here -/// would change it. Pinned by `response_carries_no_id` in `tests.rs`. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionsResponse { - pub created: u64, - pub model: String, - pub choices: Vec, - pub usage: ChatCompletionsUsage, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionToolCallFunctionChunk { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, - pub arguments: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provider_specific_fields: Option>, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionToolCallChunk { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub id: Option, - #[serde(rename = "type")] - pub tool_type: String, - pub function: ChatCompletionToolCallFunctionChunk, - pub index: i64, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ChatCompletionThinkingBlock { - Thinking { - #[serde(default, skip_serializing_if = "Option::is_none")] - thinking: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - signature: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - cache_control: Option, - }, - RedactedThinking { - #[serde(default, skip_serializing_if = "Option::is_none")] - data: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - cache_control: Option, - }, -} - -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionDelta { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub content: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub role: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tool_calls: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reasoning_content: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub thinking_blocks: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provider_specific_fields: Option>, - #[serde(flatten)] - pub extra: Map, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionStreamingChoice { - pub index: u64, - pub delta: ChatCompletionDelta, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub finish_reason: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub logprobs: Option, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionChunk { - pub id: String, - pub created: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub model: Option, - pub object: String, - pub choices: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub usage: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provider_specific_fields: Option>, -} diff --git a/litellm-rust/crates/providers/src/lib.rs b/litellm-rust/crates/providers/src/lib.rs index 5d72ffffb2b..67e65d0b8b4 100644 --- a/litellm-rust/crates/providers/src/lib.rs +++ b/litellm-rust/crates/providers/src/lib.rs @@ -1,8 +1,4 @@ pub mod anthropic; -pub mod audio_transcription; pub mod azure_ai; pub mod base_llm; pub mod bedrock; -pub mod chat; -pub mod messages; -pub mod provider_resolution; diff --git a/litellm-rust/crates/providers/src/messages/mod.rs b/litellm-rust/crates/providers/src/messages/mod.rs deleted file mode 100644 index 07232b36b51..00000000000 --- a/litellm-rust/crates/providers/src/messages/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -use thiserror::Error; - -#[derive(Clone, Debug, PartialEq, Eq, Error)] -pub enum Error { - #[error("missing required field: {0}")] - MissingField(&'static str), - #[error("invalid request: {0}")] - InvalidRequest(String), - #[error("invalid response: {0}")] - InvalidResponse(String), - #[error("unsupported: {0}")] - Unsupported(&'static str), - #[error(transparent)] - Auth(#[from] litellm_auth::Error), -} - -pub mod types; diff --git a/litellm-rust/crates/providers/src/provider_resolution.rs b/litellm-rust/crates/providers/src/provider_resolution.rs deleted file mode 100644 index d1ada2472e9..00000000000 --- a/litellm-rust/crates/providers/src/provider_resolution.rs +++ /dev/null @@ -1,33 +0,0 @@ -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CustomLlmProvider<'a> { - pub model: &'a str, - pub custom_llm_provider: &'a str, -} - -pub fn get_custom_llm_provider<'a>( - model: &'a str, - custom_llm_provider: Option<&'a str>, -) -> Option> { - if let Some(custom_llm_provider) = custom_llm_provider.filter(|provider| !provider.is_empty()) { - return Some(CustomLlmProvider { - model: strip_custom_llm_provider_prefix(model, custom_llm_provider), - custom_llm_provider, - }); - } - - let (custom_llm_provider, model) = model.split_once('/')?; - if custom_llm_provider.is_empty() || model.is_empty() { - return None; - } - Some(CustomLlmProvider { - model, - custom_llm_provider, - }) -} - -fn strip_custom_llm_provider_prefix<'a>(model: &'a str, custom_llm_provider: &str) -> &'a str { - model - .strip_prefix(custom_llm_provider) - .and_then(|model| model.strip_prefix('/')) - .unwrap_or(model) -} diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 2959fac1084..7b9818a8d4b 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -20,6 +20,7 @@ bytes.workspace = true litellm-auth.workspace = true litellm-callbacks-legacy.workspace = true litellm-core.workspace = true +litellm-types.workspace = true litellm-host-python.workspace = true litellm-token-counter.workspace = true pyo3.workspace = true diff --git a/litellm-rust/crates/python-bridge/benches/serialization.rs b/litellm-rust/crates/python-bridge/benches/serialization.rs index 7641f35932a..d398d9fdbfc 100644 --- a/litellm-rust/crates/python-bridge/benches/serialization.rs +++ b/litellm-rust/crates/python-bridge/benches/serialization.rs @@ -1,10 +1,8 @@ -use std::hint::black_box; -use std::time::Duration; +use std::{hint::black_box, time::Duration}; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use litellm_host_python::{from_py, to_py}; -use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::{prelude::*, types::PyDict}; use serde_json::{Value, json}; const PAYLOAD_SIZES: &[(&str, usize)] = &[ diff --git a/litellm-rust/crates/python-bridge/src/credentials.rs b/litellm-rust/crates/python-bridge/src/credentials.rs index 5a546f9628e..44437ec2a02 100644 --- a/litellm-rust/crates/python-bridge/src/credentials.rs +++ b/litellm-rust/crates/python-bridge/src/credentials.rs @@ -3,10 +3,12 @@ use litellm_auth::{ResolvedCredential, SecretValue}; use litellm_host_python::wrap_failure; -use pyo3::exceptions::PyTypeError; -use pyo3::gc::{PyTraverseError, PyVisit}; -use pyo3::prelude::*; -use pyo3::types::{PyDict, PyString}; +use pyo3::{ + exceptions::PyTypeError, + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::{PyDict, PyString}, +}; const NOT_CALLABLE: &str = "Azure AD token provider must be callable"; const NOT_A_STRING: &str = "Azure AD token must be a string, got {}"; diff --git a/litellm-rust/crates/python-bridge/src/diagnostics.rs b/litellm-rust/crates/python-bridge/src/diagnostics.rs index 42db4510faa..39fa8bc3596 100644 --- a/litellm-rust/crates/python-bridge/src/diagnostics.rs +++ b/litellm-rust/crates/python-bridge/src/diagnostics.rs @@ -1,6 +1,5 @@ use litellm_host_python::release_count; -use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::{prelude::*, types::PyDict}; #[pyfunction] pub(crate) fn gil_stats(py: Python<'_>) -> PyResult> { diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index 3d6f4e2a0dd..a2ea7aa2aaa 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -1,7 +1,11 @@ -use litellm_core::transport::Error as TransportError; -use litellm_core::{Error, audio_transcription, chat_completions, messages, ocr, responses}; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::*; +use litellm_core::{ + Error, audio_transcription, chat_completions, messages, ocr, responses, + transport::Error as TransportError, +}; +use pyo3::{ + exceptions::{PyRuntimeError, PyValueError}, + prelude::*, +}; pyo3::create_exception!( _native, diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index 294c439e7e9..ea4077b102f 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -1,13 +1,12 @@ -use std::collections::{BTreeMap, HashMap}; -use std::time::Duration; - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::PyDict; -use serde_json::{Map, Value}; +use std::{ + collections::{BTreeMap, HashMap}, + time::Duration, +}; use litellm_auth::InputSource; use litellm_host_python::{from_py, from_py_argument}; +use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; +use serde_json::{Map, Value}; /// The keyword arguments every value route shares, validated at the Python boundary. pub(crate) struct RouteOptions { @@ -156,10 +155,11 @@ pub(crate) fn marshal_headers(headers: Option) -> PyResult(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> { let locals = PyDict::new(py); py.run(source, Some(&locals), Some(&locals)).unwrap(); diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs index 248475b26ed..d63e9a1feaf 100644 --- a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs @@ -1,13 +1,13 @@ use litellm_core::audio_transcription::{ - AudioTranscriptionRequest, Error, audio_transcription as run_audio_transcription, + Error, audio_transcription as run_audio_transcription, types::AudioTranscriptionRequest, }; use litellm_host_python::{from_py_argument, run_async, run_sync}; use pyo3::prelude::*; use serde_json::{Map, Value}; -use crate::errors::audio_transcription_error_to_pyerr; -use crate::marshal::{ - RouteOptions, extra_headers_argument, optional_params_argument, optional_timeout, +use crate::{ + errors::audio_transcription_error_to_pyerr, + marshal::{RouteOptions, extra_headers_argument, optional_params_argument, optional_timeout}, }; async fn execute( diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs index 67036c307e2..049a507dcdc 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs @@ -1,16 +1,18 @@ -use litellm_core::chat_completions::Error; -use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompletionsResponse}; use litellm_core::chat_completions::{ - chat_completions as run_chat_completions, chat_completions_decline_reason, + Error, chat_completions as run_chat_completions, chat_completions_decline_reason, + types::ChatCompletionsRequest, }; use litellm_host_python::{from_py_argument, run_async, run_sync}; +use litellm_types::utils::ChatCompletionsResponse; use pyo3::prelude::*; use serde_json::{Map, Value}; -use crate::errors::chat_completions_error_to_pyerr; -use crate::marshal::{ - RouteOptions, extra_headers_argument, messages_argument, optional_params_argument, - optional_timeout, +use crate::{ + errors::chat_completions_error_to_pyerr, + marshal::{ + RouteOptions, extra_headers_argument, messages_argument, optional_params_argument, + optional_timeout, + }, }; async fn execute( @@ -122,8 +124,7 @@ pub(crate) fn achat_completions<'py>( #[cfg(test)] mod tests { - use pyo3::prelude::*; - use pyo3::types::PyList; + use pyo3::{prelude::*, types::PyList}; #[test] fn chat_completions_decline_keeps_existing_reasons() { diff --git a/litellm-rust/crates/python-bridge/src/routes/messages.rs b/litellm-rust/crates/python-bridge/src/routes/messages.rs index 371e8c27171..daec931c92e 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages.rs @@ -1,12 +1,13 @@ -use litellm_core::messages::Error; -use litellm_core::messages::messages as run_messages; -use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; +use litellm_core::messages::{Error, messages as run_messages, types::MessagesRequest}; use litellm_host_python::{run_async, run_sync}; +use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; use pyo3::prelude::*; use serde_json::{Map, Value}; -use crate::errors::messages_error_to_pyerr; -use crate::marshal::{RouteOptions, body_argument, extra_headers_argument, optional_timeout}; +use crate::{ + errors::messages_error_to_pyerr, + marshal::{RouteOptions, body_argument, extra_headers_argument, optional_timeout}, +}; async fn execute( body: Map, diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index b6ada947597..f59e32a28e2 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -6,8 +6,10 @@ pub(crate) mod responses; #[cfg(test)] mod tests { - use pyo3::prelude::*; - use pyo3::types::{PyDict, PyList}; + use pyo3::{ + prelude::*, + types::{PyDict, PyList}, + }; #[test] fn sync_and_async_route_signatures_match_the_python_contract() { diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs index 1a111ca2c11..7dd110cae0f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs @@ -1,13 +1,14 @@ use std::path::PathBuf; use bytes::Bytes; -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::gc::{PyTraverseError, PyVisit}; -use pyo3::prelude::*; -use pyo3::pybacked::PyBackedBytes; -use pyo3::types::{PyBytes, PyString}; - use litellm_core::ocr::{OcrDocumentInput, OcrFileContent}; +use pyo3::{ + exceptions::{PyTypeError, PyValueError}, + gc::{PyTraverseError, PyVisit}, + prelude::*, + pybacked::PyBackedBytes, + types::{PyBytes, PyString}, +}; #[derive(Debug)] pub(super) struct PythonFileReader { @@ -128,9 +129,10 @@ impl FromPyObject<'_, '_> for FileDocumentInput { #[cfg(test)] mod tests { - use super::*; use pyo3::types::PyDict; + use super::*; + fn eval<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> { let locals = PyDict::new(py); py.run(source, Some(&locals), Some(&locals)).unwrap(); diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs index 215060b7a9b..a130a95846b 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs @@ -1,6 +1,8 @@ use litellm_core::ocr::Error; -use pyo3::exceptions::{PyFileNotFoundError, PyOSError}; -use pyo3::prelude::*; +use pyo3::{ + exceptions::{PyFileNotFoundError, PyOSError}, + prelude::*, +}; use crate::errors::{RustUpstreamError, core_error_to_pyerr}; @@ -59,9 +61,10 @@ fn attach_status(error: PyErr, status: Option) -> PyErr { #[cfg(test)] mod tests { - use super::*; use pyo3::exceptions::PyValueError; + use super::*; + #[test] fn preserves_python_validation_and_provider_details() { Python::initialize(); diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs index a0f2714753d..7db83202cb7 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs @@ -1,13 +1,17 @@ use litellm_auth::ResolvedCredential; use litellm_core::ocr::{LiteLLMOcrResponse, Ocr, OcrOp, OcrOpResult}; use litellm_host_python::{RouteHost, missing_state, to_py}; -use pyo3::exceptions::PyBaseException; -use pyo3::gc::{PyTraverseError, PyVisit}; -use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::{ + exceptions::PyBaseException, + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::PyDict, +}; -use super::errors::to_pyerr as ocr_error_to_pyerr; -use super::project::{OcrHostHandles, project_request}; +use super::{ + errors::to_pyerr as ocr_error_to_pyerr, + project::{OcrHostHandles, project_request}, +}; enum OcrHostData { Unprojected, diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index 87590b52dd5..74b0629122a 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -3,12 +3,13 @@ mod errors; mod host; mod project; +use host::OcrRouteHost; use litellm_callbacks_legacy::{LegacySurface, PublicCall, run_legacy_call}; use litellm_core::ocr::{OcrClient, ocr_machine}; -use pyo3::prelude::*; -use pyo3::types::{PyDict, PyTuple}; - -use host::OcrRouteHost; +use pyo3::{ + prelude::*, + types::{PyDict, PyTuple}, +}; const SURFACE: LegacySurface = LegacySurface { call_type: "ocr", diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index 314bdec0e1b..4302fdceb0a 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -1,17 +1,19 @@ -use litellm_core::ocr::wire::{ - OcrWireRequest, consumed_optional_params, decode_document, decode_request_input, +use litellm_core::ocr::{ + LiteLLMOcrRequest, OcrDocumentInput, + wire::{OcrWireRequest, consumed_optional_params, decode_document, decode_request_input}, }; -use litellm_core::ocr::{LiteLLMOcrRequest, OcrDocumentInput}; use litellm_host_python::from_py; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; use serde_json::{Map, Value}; -use super::document::{FileDocumentInput, PythonFileReader}; -use super::errors::to_pyerr as ocr_error_to_pyerr; -use crate::credentials::{self, CallerTokenProvider}; -use crate::marshal::{project_optional_fields, python_timeout_seconds, request_input_sources}; +use super::{ + document::{FileDocumentInput, PythonFileReader}, + errors::to_pyerr as ocr_error_to_pyerr, +}; +use crate::{ + credentials::{self, CallerTokenProvider}, + marshal::{project_optional_fields, python_timeout_seconds, request_input_sources}, +}; /// What the host keeps after projection: the caller's callables that answer the document /// read and token operations, and the provider name the failure mapping reports. diff --git a/litellm-rust/crates/python-bridge/src/routes/responses.rs b/litellm-rust/crates/python-bridge/src/routes/responses.rs index bf48e4619a9..9c10d58de4f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses.rs @@ -2,8 +2,10 @@ use litellm_core::responses::websocket::ResponsesWebSocketConnection as RustResp use pyo3::prelude::*; use serde_json::Value; -use crate::errors::responses_error_to_pyerr; -use crate::marshal::{marshal_headers, optional_timeout}; +use crate::{ + errors::responses_error_to_pyerr, + marshal::{marshal_headers, optional_timeout}, +}; #[pyclass] pub(crate) struct ResponsesWebSocketConnection { @@ -58,12 +60,10 @@ impl ResponsesWebSocketConnection { #[cfg(test)] mod tests { - use std::ffi::CString; - use std::time::Duration; + use std::{ffi::CString, time::Duration}; use futures_util::{SinkExt, StreamExt}; - use pyo3::prelude::*; - use pyo3::types::PyDict; + use pyo3::{prelude::*, types::PyDict}; use tokio::net::TcpListener; use tokio_tungstenite::{accept_async, tungstenite::Message}; diff --git a/litellm-rust/crates/python-bridge/src/token_counter.rs b/litellm-rust/crates/python-bridge/src/token_counter.rs index 117e2b6e6ff..7dc86b78ad6 100644 --- a/litellm-rust/crates/python-bridge/src/token_counter.rs +++ b/litellm-rust/crates/python-bridge/src/token_counter.rs @@ -1,18 +1,17 @@ -use std::num::NonZero; -use std::sync::Arc; -use std::thread::available_parallelism; +use std::{num::NonZero, sync::Arc, thread::available_parallelism}; -use litellm_host_python::release_gil; +use litellm_host_python::{release_gil, run_async}; use litellm_token_counter::{ CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, }; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::*; -use pyo3::types::PyAny; +use pyo3::{ + exceptions::{PyRuntimeError, PyValueError}, + prelude::*, + types::PyAny, +}; use tokio::sync::Semaphore; use crate::errors::RustBridgeDeclined; -use litellm_host_python::run_async; /// Counts the input tokens of a raw request body off the Python event loop with /// the GIL released. Python owns which requests get here and what to do with diff --git a/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs b/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs index e99c01ae57e..86809ecded9 100644 --- a/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs +++ b/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs @@ -1,5 +1,7 @@ -use std::fs; -use std::path::{Path, PathBuf}; +use std::{ + fs, + path::{Path, PathBuf}, +}; const DISALLOWED_OUTSIDE_INTEROP: &[&str] = &[ "py.import(\"json\")", diff --git a/litellm-rust/crates/types/Cargo.toml b/litellm-rust/crates/types/Cargo.toml new file mode 100644 index 00000000000..6a2efa90ab4 --- /dev/null +++ b/litellm-rust/crates/types/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "litellm-types" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +serde.workspace = true +serde_json.workspace = true diff --git a/litellm-rust/crates/types/src/lib.rs b/litellm-rust/crates/types/src/lib.rs new file mode 100644 index 00000000000..da5c9ea893f --- /dev/null +++ b/litellm-rust/crates/types/src/lib.rs @@ -0,0 +1,3 @@ +pub mod llms; +pub mod responses; +pub mod utils; diff --git a/litellm-rust/crates/providers/src/messages/types.rs b/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_request.rs similarity index 69% rename from litellm-rust/crates/providers/src/messages/types.rs rename to litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_request.rs index ba274ab9651..50eedf7ba09 100644 --- a/litellm-rust/crates/providers/src/messages/types.rs +++ b/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_request.rs @@ -1,30 +1,6 @@ -use std::time::Duration; - use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use crate::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig; - -pub struct MessagesRequest<'a> { - pub model: &'a str, - pub body: Value, - pub api_key: Option<&'a str>, - pub api_base: Option<&'a str>, - pub custom_llm_provider: Option<&'a str>, - pub extra_headers: Option>, - pub timeout: Option, -} - -pub struct ProviderMessagesRequest { - pub provider: String, - pub model: String, - pub config: &'static dyn BaseAnthropicMessagesConfig, - pub url: String, - pub body: Value, - pub upstream_headers: Vec<(String, String)>, - pub timeout: Option, -} - #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(untagged)] pub enum SystemPrompt { @@ -112,23 +88,3 @@ pub struct AnthropicMessagesRequest { #[serde(flatten)] pub extra: Map, } - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct AnthropicMessagesResponse { - pub id: String, - #[serde(rename = "type")] - pub message_type: String, - pub role: String, - pub model: String, - pub content: Vec, - // Anthropic always includes stop_reason / stop_sequence, null until the turn - // ends; serialize them even when None so callers see the same shape as Python. - pub stop_reason: Option, - pub stop_sequence: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub usage: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub container: Option, - #[serde(flatten)] - pub extra: Map, -} diff --git a/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_response.rs b/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_response.rs new file mode 100644 index 00000000000..0c3876aac59 --- /dev/null +++ b/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_response.rs @@ -0,0 +1,22 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct AnthropicMessagesResponse { + pub id: String, + #[serde(rename = "type")] + pub message_type: String, + pub role: String, + pub model: String, + pub content: Vec, + // Anthropic always includes stop_reason / stop_sequence, null until the turn + // ends; serialize them even when None so callers see the same shape as Python. + pub stop_reason: Option, + pub stop_sequence: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub container: Option, + #[serde(flatten)] + pub extra: Map, +} diff --git a/litellm-rust/crates/types/src/llms/anthropic_messages/mod.rs b/litellm-rust/crates/types/src/llms/anthropic_messages/mod.rs new file mode 100644 index 00000000000..2b6ada1f22e --- /dev/null +++ b/litellm-rust/crates/types/src/llms/anthropic_messages/mod.rs @@ -0,0 +1,2 @@ +pub mod anthropic_request; +pub mod anthropic_response; diff --git a/litellm-rust/crates/types/src/llms/mod.rs b/litellm-rust/crates/types/src/llms/mod.rs new file mode 100644 index 00000000000..09d2207a0ca --- /dev/null +++ b/litellm-rust/crates/types/src/llms/mod.rs @@ -0,0 +1,2 @@ +pub mod anthropic_messages; +pub mod openai; diff --git a/litellm-rust/crates/types/src/llms/openai.rs b/litellm-rust/crates/types/src/llms/openai.rs new file mode 100644 index 00000000000..232f5b9cc51 --- /dev/null +++ b/litellm-rust/crates/types/src/llms/openai.rs @@ -0,0 +1,58 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ChatMessageContent { + Text(String), + Parts(Vec), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatMessage { + pub role: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionToolCallFunctionChunk { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub arguments: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option>, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionToolCallChunk { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(rename = "type")] + pub tool_type: String, + pub function: ChatCompletionToolCallFunctionChunk, + pub index: i64, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ChatCompletionThinkingBlock { + Thinking { + #[serde(default, skip_serializing_if = "Option::is_none")] + thinking: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + signature: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cache_control: Option, + }, + RedactedThinking { + #[serde(default, skip_serializing_if = "Option::is_none")] + data: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cache_control: Option, + }, +} diff --git a/litellm-rust/crates/types/src/responses/mod.rs b/litellm-rust/crates/types/src/responses/mod.rs new file mode 100644 index 00000000000..02493c5f6ed --- /dev/null +++ b/litellm-rust/crates/types/src/responses/mod.rs @@ -0,0 +1 @@ +pub mod streaming_websocket; diff --git a/litellm-rust/crates/core/src/responses/types.rs b/litellm-rust/crates/types/src/responses/streaming_websocket.rs similarity index 100% rename from litellm-rust/crates/core/src/responses/types.rs rename to litellm-rust/crates/types/src/responses/streaming_websocket.rs diff --git a/litellm-rust/crates/types/src/utils.rs b/litellm-rust/crates/types/src/utils.rs new file mode 100644 index 00000000000..7f0c18f9f2c --- /dev/null +++ b/litellm-rust/crates/types/src/utils.rs @@ -0,0 +1,93 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use crate::llms::openai::{ChatCompletionThinkingBlock, ChatCompletionToolCallChunk}; + +/// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python +/// path reports so cost tracking sees the same numbers on either path. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct PromptTokensDetails { + pub cached_tokens: u64, + pub cache_creation_tokens: u64, + pub text_tokens: u64, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionsUsage { + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub total_tokens: u64, + pub prompt_tokens_details: PromptTokensDetails, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionsChoiceMessage { + pub role: String, + // Whether an empty turn is `None` or `""` is the provider's choice, not a + // shared invariant: Anthropic's transform ends on `merged_text or None` + // while Converse assigns the joined string unconditionally. Each config + // mirrors its own, so keep this optional and serialize it even when None. + pub content: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionsChoice { + pub index: u64, + pub message: ChatCompletionsChoiceMessage, + pub finish_reason: String, +} + +/// The normalized response handed back to the host. +/// +/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the +/// `ModelResponse` it already created, and echoing the provider's own id here +/// would change it. Pinned by `response_carries_no_id` in `tests.rs`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionsResponse { + pub created: u64, + pub model: String, + pub choices: Vec, + pub usage: ChatCompletionsUsage, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionDelta { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thinking_blocks: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionStreamingChoice { + pub index: u64, + pub delta: ChatCompletionDelta, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finish_reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logprobs: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionChunk { + pub id: String, + pub created: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model: Option, + pub object: String, + pub choices: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option>, +}