diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 196f282b5fb..075ee076658 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2025,8 +2025,6 @@ dependencies = [ name = "litellm-core" version = "0.1.0" dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-types", "base64 0.22.1", "bytes", "data-url", @@ -2037,8 +2035,7 @@ dependencies = [ "litellm-auth-gcp", "litellm-callbacks", "litellm-core-utils", - "litellm-framing", - "litellm-providers", + "litellm-llms", "litellm-types", "mime_guess", "moka", @@ -2107,17 +2104,26 @@ dependencies = [ ] [[package]] -name = "litellm-providers" +name = "litellm-llms" version = "0.1.0" dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-types", + "base64 0.22.1", + "bytes", + "futures-util", "litellm-auth", "litellm-auth-aws", "litellm-core-utils", + "litellm-framing", "litellm-types", "rstest", "serde", "serde_json", "thiserror 2.0.19", + "time", + "tokio", + "url", ] [[package]] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 89b217cadda..f8377138050 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -17,7 +17,7 @@ litellm-auth = { path = "crates/auth" } litellm-auth-aws = { path = "crates/auth-aws" } litellm-auth-azure = { path = "crates/auth-azure" } litellm-auth-gcp = { path = "crates/auth-gcp" } -litellm-providers = { path = "crates/providers" } +litellm-llms = { path = "crates/llms" } litellm-types = { path = "crates/types" } litellm-core-utils = { path = "crates/core-utils" } litellm-cache = { path = "crates/cache" } diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 33f44d66bb2..7695e257bcd 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -18,8 +18,7 @@ litellm-auth.workspace = true litellm-auth-aws.workspace = true litellm-auth-azure.workspace = true litellm-auth-gcp.workspace = true -litellm-providers.workspace = true -litellm-framing.workspace = true +litellm-llms.workspace = true moka.workspace = true mime_guess = "2.0.5" rand.workspace = true @@ -41,7 +40,5 @@ url.workspace = true veil.workspace = true [dev-dependencies] -aws-smithy-eventstream = "=0.61.1" -aws-smithy-types = "1.6.1" rstest.workspace = true rstest_reuse.workspace = true diff --git a/litellm-rust/crates/core/src/audio_transcription/error.rs b/litellm-rust/crates/core/src/audio_transcription/error.rs index cfa9e4cc5e2..6c08727795b 100644 --- a/litellm-rust/crates/core/src/audio_transcription/error.rs +++ b/litellm-rust/crates/core/src/audio_transcription/error.rs @@ -1,4 +1,4 @@ -use litellm_providers::base_llm::chat::transformation::Error as LlmError; +use litellm_llms::base_llm::chat::transformation::Error as LlmError; #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { diff --git a/litellm-rust/crates/core/src/audio_transcription/handler.rs b/litellm-rust/crates/core/src/audio_transcription/handler.rs index e79833cf201..01cf236e118 100644 --- a/litellm-rust/crates/core/src/audio_transcription/handler.rs +++ b/litellm-rust/crates/core/src/audio_transcription/handler.rs @@ -48,7 +48,7 @@ async fn signed_headers( use std::{collections::BTreeMap, time::SystemTime}; use litellm_auth_aws::{aws_auth_config, resolve_credentials, sign_bedrock_post}; - use litellm_providers::base_llm::audio_transcription::transformation::AudioTranscriptionAuth; + use litellm_llms::base_llm::audio_transcription::transformation::AudioTranscriptionAuth; let AudioTranscriptionAuth::AwsSigV4 { region, .. } = &request.auth else { return Ok(request.upstream_headers.clone()); diff --git a/litellm-rust/crates/core/src/audio_transcription/prepare.rs b/litellm-rust/crates/core/src/audio_transcription/prepare.rs index 0e230117834..25c9afd5ea3 100644 --- a/litellm-rust/crates/core/src/audio_transcription/prepare.rs +++ b/litellm-rust/crates/core/src/audio_transcription/prepare.rs @@ -1,5 +1,5 @@ use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; -use litellm_providers::{ +use litellm_llms::{ base_llm::audio_transcription::transformation::{ AudioTranscriptionAuth, BaseAudioTranscriptionConfig, }, diff --git a/litellm-rust/crates/core/src/audio_transcription/types.rs b/litellm-rust/crates/core/src/audio_transcription/types.rs index 7878e4bd175..ca09dd945be 100644 --- a/litellm-rust/crates/core/src/audio_transcription/types.rs +++ b/litellm-rust/crates/core/src/audio_transcription/types.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use litellm_providers::base_llm::audio_transcription::transformation::{ +use litellm_llms::base_llm::audio_transcription::transformation::{ AudioTranscriptionAuth, BaseAudioTranscriptionConfig, }; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/core/src/chat_completions/common_utils.rs b/litellm-rust/crates/core/src/chat_completions/common_utils.rs index 309cc781cc0..bb524db661e 100644 --- a/litellm-rust/crates/core/src/chat_completions/common_utils.rs +++ b/litellm-rust/crates/core/src/chat_completions/common_utils.rs @@ -1,4 +1,4 @@ -use litellm_providers::{ +use litellm_llms::{ anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG, base_llm::chat::transformation::BaseConfig, }; @@ -13,7 +13,7 @@ pub(super) fn chat_completions_provider_config(provider: &str) -> Option<&'stati match provider { "anthropic" => Some(&ANTHROPIC_CHAT_COMPLETIONS_CONFIG), "bedrock" => Some( - &litellm_providers::bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, + &litellm_llms::bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, ), _ => None, } diff --git a/litellm-rust/crates/core/src/chat_completions/error.rs b/litellm-rust/crates/core/src/chat_completions/error.rs index cfa9e4cc5e2..6c08727795b 100644 --- a/litellm-rust/crates/core/src/chat_completions/error.rs +++ b/litellm-rust/crates/core/src/chat_completions/error.rs @@ -1,4 +1,4 @@ -use litellm_providers::base_llm::chat::transformation::Error as LlmError; +use litellm_llms::base_llm::chat::transformation::Error as LlmError; #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index ae8c0dd6144..664841318fc 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -1,6 +1,4 @@ -use litellm_providers::base_llm::chat::transformation::{ - ChatCompletionsAuth, ProviderChatResponseData, -}; +use litellm_llms::base_llm::chat::transformation::{ChatCompletionsAuth, ProviderChatResponseData}; use litellm_types::utils::ChatCompletionsResponse; use serde_json::Value; diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index 5c5cc48692a..81d35044d08 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -13,7 +13,6 @@ mod client; mod common_utils; pub(crate) mod handler; mod prepare; -pub mod streaming; use handler::execute_chat_completions_provider_call; use litellm_types::utils::ChatCompletionsResponse; use prepare::{parse_messages, resolve_provider_config, resolve_request}; diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/core/src/chat_completions/prepare.rs index 882da4e584b..4dae9af966c 100644 --- a/litellm-rust/crates/core/src/chat_completions/prepare.rs +++ b/litellm-rust/crates/core/src/chat_completions/prepare.rs @@ -1,5 +1,5 @@ use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; -use litellm_providers::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; +use litellm_llms::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; use litellm_types::llms::openai::ChatMessage; use serde_json::Value; diff --git a/litellm-rust/crates/core/src/chat_completions/tests.rs b/litellm-rust/crates/core/src/chat_completions/tests.rs index b8bf8625e9c..b52cc538a65 100644 --- a/litellm-rust/crates/core/src/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/chat_completions/tests.rs @@ -1,4 +1,4 @@ -use litellm_providers::base_llm::chat::transformation::ChatCompletionsAuth; +use litellm_llms::base_llm::chat::transformation::ChatCompletionsAuth; use serde_json::{Map, Value, json}; use super::{ diff --git a/litellm-rust/crates/core/src/chat_completions/types.rs b/litellm-rust/crates/core/src/chat_completions/types.rs index 1687ef0a792..882611d5862 100644 --- a/litellm-rust/crates/core/src/chat_completions/types.rs +++ b/litellm-rust/crates/core/src/chat_completions/types.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use litellm_providers::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; +use litellm_llms::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; use litellm_types::llms::openai::ChatMessage; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index 4ff4333c4ac..607ce32c6d4 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -1,6 +1,4 @@ pub const OPENAI_DEFAULT_API_BASE: &str = "https://api.openai.com"; -pub const OPENAI_RESPONSES_DEFAULT_API_BASE: &str = "https://api.openai.com/v1"; -pub const OPENAI_RESPONSES_PATH: &str = "/responses"; /// Full-request timeout ceiling for Anthropic Messages provider calls, in /// seconds. Mirrors the Python Anthropic Messages default. The per-request @@ -18,11 +16,6 @@ pub(crate) const UPSTREAM_ERROR_BODY_MAX_CHARS: usize = 256; /// does not carry an explicit provider prefix. pub const ANTHROPIC_MESSAGES_PROVIDER: &str = "anthropic"; -/// Prefix identifying an Anthropic OAuth token. Mirrors Python's -/// `ANTHROPIC_OAUTH_TOKEN_PREFIX`, which is what makes `validate_environment` -/// authenticate with `authorization` and drop `x-api-key` entirely. -pub(crate) const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat"; - /// Full-request timeout ceiling for chat completions provider calls, in /// seconds. Mirrors the Python chat completions default. pub(crate) const CHAT_COMPLETIONS_TIMEOUT_SECS: u64 = 600; diff --git a/litellm-rust/crates/core/src/llms/anthropic/chat/mod.rs b/litellm-rust/crates/core/src/llms/anthropic/chat/mod.rs deleted file mode 100644 index 7bf4fc46291..00000000000 --- a/litellm-rust/crates/core/src/llms/anthropic/chat/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod streaming; diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/mod.rs b/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/mod.rs deleted file mode 100644 index 42d4fcdde0f..00000000000 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod batches; -pub mod count_tokens; -pub mod streaming; diff --git a/litellm-rust/crates/core/src/llms/anthropic/mod.rs b/litellm-rust/crates/core/src/llms/anthropic/mod.rs deleted file mode 100644 index 4943d80a45c..00000000000 --- a/litellm-rust/crates/core/src/llms/anthropic/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod chat; -pub mod experimental_pass_through; diff --git a/litellm-rust/crates/core/src/llms/mod.rs b/litellm-rust/crates/core/src/llms/mod.rs index 4b93a5f971c..8b0a67f6a94 100644 --- a/litellm-rust/crates/core/src/llms/mod.rs +++ b/litellm-rust/crates/core/src/llms/mod.rs @@ -1,8 +1,6 @@ -pub mod anthropic; pub mod azure_ai; pub mod base_llm; pub(crate) mod cohere; pub(crate) mod mistral; -pub mod openai; pub(crate) mod reducto; pub(crate) mod vertex_ai; diff --git a/litellm-rust/crates/core/src/messages/common_utils.rs b/litellm-rust/crates/core/src/messages/common_utils.rs index 81d67520abe..f73f62c471c 100644 --- a/litellm-rust/crates/core/src/messages/common_utils.rs +++ b/litellm-rust/crates/core/src/messages/common_utils.rs @@ -1,4 +1,4 @@ -use litellm_providers::{ +use litellm_llms::{ anthropic::experimental_pass_through::messages::transformation::ANTHROPIC_MESSAGES_CONFIG, azure_ai::anthropic::messages_transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG, base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig, diff --git a/litellm-rust/crates/core/src/messages/error.rs b/litellm-rust/crates/core/src/messages/error.rs index 7a9c1c1d20a..58a88b6d16e 100644 --- a/litellm-rust/crates/core/src/messages/error.rs +++ b/litellm-rust/crates/core/src/messages/error.rs @@ -1,4 +1,4 @@ -use litellm_providers::base_llm::chat::transformation::Error as LlmError; +use litellm_llms::base_llm::chat::transformation::Error as LlmError; #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { @@ -18,16 +18,6 @@ pub enum Error { Transport(#[from] crate::transport::Error), #[error(transparent)] Headers(#[from] crate::http_utils::HeaderError), - #[error("stream framing failed: {0}")] - StreamFraming(String), - #[error("Anthropic SSE frame has no data")] - MissingStreamData, - #[error("Anthropic stream event is invalid: {0}")] - InvalidStreamEvent(String), - #[error("Bedrock event payload is invalid: {0}")] - InvalidBedrockPayload(String), - #[error("Bedrock event payload has invalid base64: {0}")] - InvalidBedrockBase64(String), } impl From for Error { @@ -57,14 +47,6 @@ impl Error { } pub fn is_response(&self) -> bool { - matches!( - self, - Self::InvalidResponse(_) - | Self::StreamFraming(_) - | Self::MissingStreamData - | Self::InvalidStreamEvent(_) - | Self::InvalidBedrockPayload(_) - | Self::InvalidBedrockBase64(_) - ) + matches!(self, Self::InvalidResponse(_)) } } diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index d7e996d74e8..8b676803871 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,5 +1,5 @@ use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; -use litellm_providers::base_llm::anthropic_messages::transformation::{ +use litellm_llms::base_llm::anthropic_messages::transformation::{ BaseAnthropicMessagesConfig, MessagesAuthStrategy, }; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/core/src/messages/types.rs b/litellm-rust/crates/core/src/messages/types.rs index 928eca222be..a73ceffad7a 100644 --- a/litellm-rust/crates/core/src/messages/types.rs +++ b/litellm-rust/crates/core/src/messages/types.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use litellm_providers::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig; +use litellm_llms::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig; use serde_json::{Map, Value}; pub struct MessagesRequest<'a> { diff --git a/litellm-rust/crates/core/src/responses/websocket.rs b/litellm-rust/crates/core/src/responses/websocket.rs index a91fbcf272f..b601af74d55 100644 --- a/litellm-rust/crates/core/src/responses/websocket.rs +++ b/litellm-rust/crates/core/src/responses/websocket.rs @@ -6,9 +6,7 @@ use std::{ }; use futures_util::{SinkExt, StreamExt}; -use litellm_types::responses::streaming_websocket::{ - ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult, -}; +use litellm_types::responses::streaming_websocket::ResponsesWsEventType; use rustls::{ClientConfig, RootCertStore}; use tokio::{net::TcpStream, sync::Mutex}; use tokio_tungstenite::{ @@ -23,119 +21,6 @@ use tokio_tungstenite::{ }; use super::Error; -use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH}; - -pub trait ResponsesWebSocketProviderConfig: Sync { - fn supports_native_websocket(&self) -> bool { - false - } - - fn model_in_websocket_url(&self) -> bool { - true - } - - fn complete_websocket_url(&self, api_base: Option<&str>, model: &str) -> String { - complete_websocket_url(api_base, model, self.model_in_websocket_url()) - } - - fn transform_ws_request( - &self, - event: &ResponsesWsEvent, - model: &str, - ) -> Result; - - fn transform_ws_response( - &self, - event: &ResponsesWsEvent, - model: &str, - ) -> Result; -} - -pub fn complete_websocket_url( - api_base: Option<&str>, - model: &str, - model_in_websocket_url: bool, -) -> String { - let base = api_base - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or(OPENAI_RESPONSES_DEFAULT_API_BASE); - let (base_without_query, query) = base - .split_once('?') - .map_or((base, None), |(value, query)| (value, Some(query))); - let response_url = format!( - "{}{}", - base_without_query.trim_end_matches('/'), - OPENAI_RESPONSES_PATH - ); - let scheme_flipped = if let Some(rest) = response_url.strip_prefix("https://") { - format!("wss://{rest}") - } else if let Some(rest) = response_url.strip_prefix("http://") { - format!("ws://{rest}") - } else { - response_url - }; - let url = query.map_or(scheme_flipped.clone(), |value| { - format!("{scheme_flipped}?{value}") - }); - if !model_in_websocket_url - || query.is_some_and(|value| { - value - .split('&') - .any(|part| part.split('=').next() == Some("model")) - }) - { - return url; - } - format!( - "{url}{}model={}", - if query.is_some() { "&" } else { "?" }, - percent_encode(model) - ) -} - -fn percent_encode(value: &str) -> String { - value - .bytes() - .map(|byte| { - if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { - format!("{}", byte as char) - } else { - format!("%{byte:02X}") - } - }) - .collect() -} - -pub fn enforce_model(event: &ResponsesWsEvent, model: &str) -> ResponsesWsEvent { - if !event.is_response_create() { - return event.clone(); - } - let mut enforced = event.clone(); - let has_flat_model = enforced.data.contains_key("model"); - if let Some(response) = enforced - .data - .get_mut("response") - .and_then(serde_json::Value::as_object_mut) - { - response.insert( - "model".to_string(), - serde_json::Value::String(model.to_string()), - ); - if has_flat_model { - enforced.data.insert( - "model".to_string(), - serde_json::Value::String(model.to_string()), - ); - } - } else { - enforced.data.insert( - "model".to_string(), - serde_json::Value::String(model.to_string()), - ); - } - enforced -} pub fn is_terminal_event(event_type: &ResponsesWsEventType) -> bool { matches!( @@ -285,65 +170,3 @@ impl ResponsesWebSocketConnection { Ok(()) } } - -#[cfg(test)] -mod tests { - use super::*; - - fn event(value: serde_json::Value) -> ResponsesWsEvent { - serde_json::from_value(value).expect("valid event") - } - - #[test] - fn url_construction_matches_python_defaults_and_query_behavior() { - assert_eq!( - complete_websocket_url(None, "gpt-5", true), - "wss://api.openai.com/v1/responses?model=gpt-5" - ); - assert_eq!( - complete_websocket_url(Some("http://localhost:8080/"), "gpt 5", true), - "ws://localhost:8080/responses?model=gpt%205" - ); - assert_eq!( - complete_websocket_url(Some("https://example.test/v1?foo=bar"), "gpt-5", true), - "wss://example.test/v1/responses?foo=bar&model=gpt-5" - ); - assert_eq!( - complete_websocket_url(Some("https://example.test?model=existing"), "gpt-5", true), - "wss://example.test/responses?model=existing" - ); - } - - #[test] - fn enforce_model_overrides_flat_and_nested_values() { - let flat = enforce_model( - &event(serde_json::json!({"type":"response.create","model":"wrong"})), - "gpt-5", - ); - assert_eq!(flat.model(), Some("gpt-5")); - let nested = enforce_model( - &event(serde_json::json!({ - "type":"response.create", - "model":"wrong", - "response":{"model":"also-wrong"} - })), - "gpt-5", - ); - assert_eq!(nested.model(), Some("gpt-5")); - assert_eq!( - nested - .data - .get("response") - .and_then(|value| value.get("model")), - Some(&serde_json::json!("gpt-5")) - ); - let nested_without_flat = enforce_model( - &event(serde_json::json!({ - "type":"response.create", - "response":{"model":"also-wrong"} - })), - "gpt-5", - ); - assert!(!nested_without_flat.data.contains_key("model")); - } -} diff --git a/litellm-rust/crates/providers/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml similarity index 59% rename from litellm-rust/crates/providers/Cargo.toml rename to litellm-rust/crates/llms/Cargo.toml index f137e2ddbe4..c7564044fcb 100644 --- a/litellm-rust/crates/providers/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "litellm-providers" +name = "litellm-llms" version = "0.1.0" edition.workspace = true license.workspace = true @@ -10,9 +10,18 @@ litellm-types.workspace = true litellm-core-utils.workspace = true litellm-auth.workspace = true litellm-auth-aws.workspace = true +litellm-framing.workspace = true +base64.workspace = true +bytes.workspace = true +futures-util.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true +time.workspace = true +url.workspace = true [dev-dependencies] +aws-smithy-eventstream = "=0.61.1" +aws-smithy-types = "1.6.1" rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/core/src/llms/openai/responses/mod.rs b/litellm-rust/crates/llms/src/anthropic/batches/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/llms/openai/responses/mod.rs rename to litellm-rust/crates/llms/src/anthropic/batches/mod.rs diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs b/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs similarity index 98% rename from litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs rename to litellm-rust/crates/llms/src/anthropic/batches/transformation.rs index e371a24c057..94e4dc7838a 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs +++ b/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs @@ -1,11 +1,13 @@ -use litellm_providers::anthropic::experimental_pass_through::messages::transformation::resolve_anthropic_api_base; use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; use serde::{Deserialize, Serialize}; use serde_json::Value; use time::OffsetDateTime; use url::Url; -use crate::messages::Error; +use crate::{ + anthropic::experimental_pass_through::messages::transformation::resolve_anthropic_api_base, + base_llm::chat::transformation::Error, +}; const BATCHES_PATH_SUFFIX: &str = "/v1/messages/batches"; diff --git a/litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs b/litellm-rust/crates/llms/src/anthropic/chat/handler.rs similarity index 93% rename from litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs rename to litellm-rust/crates/llms/src/anthropic/chat/handler.rs index 86c6e010e84..a80cfbf28bd 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/handler.rs @@ -6,11 +6,13 @@ use litellm_types::{ }; use serde_json::Value; -use super::super::experimental_pass_through::messages::streaming::{ - AnthropicContentBlock, AnthropicContentBlockDelta, AnthropicMessagesStreamEvent, - AnthropicStreamUsage, +use crate::{ + anthropic::experimental_pass_through::messages::streaming_iterator::{ + AnthropicContentBlock, AnthropicContentBlockDelta, AnthropicMessagesStreamEvent, + AnthropicStreamUsage, + }, + base_llm::{base_model_iterator::StreamTransformer, chat::transformation::Error}, }; -use crate::chat_completions::{Error, streaming::StreamTransformer}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum AnthropicJsonChunkType { diff --git a/litellm-rust/crates/llms/src/anthropic/chat/mod.rs b/litellm-rust/crates/llms/src/anthropic/chat/mod.rs new file mode 100644 index 00000000000..f0050b7dc71 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/chat/mod.rs @@ -0,0 +1,2 @@ +pub mod handler; +pub mod transformation; diff --git a/litellm-rust/crates/providers/src/anthropic/chat/tests.rs b/litellm-rust/crates/llms/src/anthropic/chat/tests.rs similarity index 100% rename from litellm-rust/crates/providers/src/anthropic/chat/tests.rs rename to litellm-rust/crates/llms/src/anthropic/chat/tests.rs diff --git a/litellm-rust/crates/providers/src/anthropic/chat/transformation.rs b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/anthropic/chat/transformation.rs rename to litellm-rust/crates/llms/src/anthropic/chat/transformation.rs diff --git a/litellm-rust/crates/providers/src/anthropic/chat/mod.rs b/litellm-rust/crates/llms/src/anthropic/count_tokens/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/anthropic/chat/mod.rs rename to litellm-rust/crates/llms/src/anthropic/count_tokens/mod.rs diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs b/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs similarity index 98% rename from litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs rename to litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs index 82a2a54c961..a4d8c57ca4f 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs +++ b/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs @@ -2,7 +2,7 @@ use litellm_types::llms::anthropic_messages::anthropic_request::{AnthropicMessag use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::{constants::ANTHROPIC_OAUTH_TOKEN_PREFIX, messages::Error}; +use crate::{anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX, base_llm::chat::transformation::Error}; const COUNT_TOKENS_ENDPOINT: &str = "https://api.anthropic.com/v1/messages/count_tokens"; const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01"; diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs new file mode 100644 index 00000000000..481d98c4e9d --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs @@ -0,0 +1,2 @@ +pub mod streaming_iterator; +pub mod transformation; diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/streaming.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/streaming_iterator.rs similarity index 94% rename from litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/streaming.rs rename to litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/streaming_iterator.rs index 92a36265df7..35e7d5820b0 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/streaming.rs +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/streaming_iterator.rs @@ -9,7 +9,19 @@ use litellm_framing::{ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use crate::messages::Error; +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("stream framing failed: {0}")] + StreamFraming(String), + #[error("Anthropic SSE frame has no data")] + MissingStreamData, + #[error("Anthropic stream event is invalid: {0}")] + InvalidStreamEvent(String), + #[error("Bedrock event payload is invalid: {0}")] + InvalidBedrockPayload(String), + #[error("Bedrock event payload has invalid base64: {0}")] + InvalidBedrockBase64(String), +} #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct AnthropicStreamUsage { diff --git a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/transformation.rs rename to litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/transformation.rs diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/mod.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/mod.rs rename to litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs diff --git a/litellm-rust/crates/providers/src/anthropic/mod.rs b/litellm-rust/crates/llms/src/anthropic/mod.rs similarity index 74% rename from litellm-rust/crates/providers/src/anthropic/mod.rs rename to litellm-rust/crates/llms/src/anthropic/mod.rs index 38a59aa6e0d..d181ceaca3c 100644 --- a/litellm-rust/crates/providers/src/anthropic/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/mod.rs @@ -1,4 +1,6 @@ +pub mod batches; pub mod chat; +pub mod count_tokens; pub mod experimental_pass_through; pub const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat"; diff --git a/litellm-rust/crates/providers/src/azure_ai/anthropic/messages_transformation.rs b/litellm-rust/crates/llms/src/azure_ai/anthropic/messages_transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/azure_ai/anthropic/messages_transformation.rs rename to litellm-rust/crates/llms/src/azure_ai/anthropic/messages_transformation.rs diff --git a/litellm-rust/crates/providers/src/azure_ai/anthropic/mod.rs b/litellm-rust/crates/llms/src/azure_ai/anthropic/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/azure_ai/anthropic/mod.rs rename to litellm-rust/crates/llms/src/azure_ai/anthropic/mod.rs diff --git a/litellm-rust/crates/providers/src/azure_ai/mod.rs b/litellm-rust/crates/llms/src/azure_ai/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/azure_ai/mod.rs rename to litellm-rust/crates/llms/src/azure_ai/mod.rs diff --git a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/mod.rs b/litellm-rust/crates/llms/src/base_llm/anthropic_messages/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/mod.rs rename to litellm-rust/crates/llms/src/base_llm/anthropic_messages/mod.rs diff --git a/litellm-rust/crates/providers/src/base_llm/anthropic_messages/transformation.rs b/litellm-rust/crates/llms/src/base_llm/anthropic_messages/transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/anthropic_messages/transformation.rs rename to litellm-rust/crates/llms/src/base_llm/anthropic_messages/transformation.rs diff --git a/litellm-rust/crates/providers/src/base_llm/anthropic_messages/mod.rs b/litellm-rust/crates/llms/src/base_llm/audio_transcription/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/anthropic_messages/mod.rs rename to litellm-rust/crates/llms/src/base_llm/audio_transcription/mod.rs diff --git a/litellm-rust/crates/providers/src/base_llm/audio_transcription/transformation.rs b/litellm-rust/crates/llms/src/base_llm/audio_transcription/transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/audio_transcription/transformation.rs rename to litellm-rust/crates/llms/src/base_llm/audio_transcription/transformation.rs diff --git a/litellm-rust/crates/core/src/chat_completions/streaming.rs b/litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs similarity index 100% rename from litellm-rust/crates/core/src/chat_completions/streaming.rs rename to litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs diff --git a/litellm-rust/crates/providers/src/base_llm/audio_transcription/mod.rs b/litellm-rust/crates/llms/src/base_llm/chat/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/audio_transcription/mod.rs rename to litellm-rust/crates/llms/src/base_llm/chat/mod.rs diff --git a/litellm-rust/crates/providers/src/base_llm/chat/transformation.rs b/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/chat/transformation.rs rename to litellm-rust/crates/llms/src/base_llm/chat/transformation.rs diff --git a/litellm-rust/crates/providers/src/base_llm/mod.rs b/litellm-rust/crates/llms/src/base_llm/mod.rs similarity index 59% rename from litellm-rust/crates/providers/src/base_llm/mod.rs rename to litellm-rust/crates/llms/src/base_llm/mod.rs index b7a1f696440..0c3aa2091a3 100644 --- a/litellm-rust/crates/providers/src/base_llm/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/mod.rs @@ -1,3 +1,5 @@ pub mod anthropic_messages; pub mod audio_transcription; +pub mod base_model_iterator; pub mod chat; +pub mod responses; diff --git a/litellm-rust/crates/providers/src/base_llm/chat/mod.rs b/litellm-rust/crates/llms/src/base_llm/responses/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/chat/mod.rs rename to litellm-rust/crates/llms/src/base_llm/responses/mod.rs diff --git a/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs b/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs new file mode 100644 index 00000000000..0d9cfcfd4cd --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs @@ -0,0 +1,180 @@ +use litellm_types::responses::streaming_websocket::{ResponsesWsEvent, ResponsesWsTransformResult}; + +use crate::base_llm::chat::transformation::Error; + +pub const OPENAI_RESPONSES_DEFAULT_API_BASE: &str = "https://api.openai.com/v1"; +pub const OPENAI_RESPONSES_PATH: &str = "/responses"; + +pub trait ResponsesWebSocketProviderConfig: Sync { + fn supports_native_websocket(&self) -> bool { + false + } + + fn model_in_websocket_url(&self) -> bool { + true + } + + fn complete_websocket_url(&self, api_base: Option<&str>, model: &str) -> String { + complete_websocket_url(api_base, model, self.model_in_websocket_url()) + } + + fn transform_ws_request( + &self, + event: &ResponsesWsEvent, + model: &str, + ) -> Result; + + fn transform_ws_response( + &self, + event: &ResponsesWsEvent, + model: &str, + ) -> Result; +} + +pub fn complete_websocket_url( + api_base: Option<&str>, + model: &str, + model_in_websocket_url: bool, +) -> String { + let base = api_base + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or(OPENAI_RESPONSES_DEFAULT_API_BASE); + let (base_without_query, query) = base + .split_once('?') + .map_or((base, None), |(value, query)| (value, Some(query))); + let response_url = format!( + "{}{}", + base_without_query.trim_end_matches('/'), + OPENAI_RESPONSES_PATH + ); + let scheme_flipped = if let Some(rest) = response_url.strip_prefix("https://") { + format!("wss://{rest}") + } else if let Some(rest) = response_url.strip_prefix("http://") { + format!("ws://{rest}") + } else { + response_url + }; + let url = query.map_or(scheme_flipped.clone(), |value| { + format!("{scheme_flipped}?{value}") + }); + if !model_in_websocket_url + || query.is_some_and(|value| { + value + .split('&') + .any(|part| part.split('=').next() == Some("model")) + }) + { + return url; + } + format!( + "{url}{}model={}", + if query.is_some() { "&" } else { "?" }, + percent_encode(model) + ) +} + +fn percent_encode(value: &str) -> String { + value + .bytes() + .map(|byte| { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + format!("{}", byte as char) + } else { + format!("%{byte:02X}") + } + }) + .collect() +} + +pub fn enforce_model(event: &ResponsesWsEvent, model: &str) -> ResponsesWsEvent { + if !event.is_response_create() { + return event.clone(); + } + let mut enforced = event.clone(); + let has_flat_model = enforced.data.contains_key("model"); + if let Some(response) = enforced + .data + .get_mut("response") + .and_then(serde_json::Value::as_object_mut) + { + response.insert( + "model".to_string(), + serde_json::Value::String(model.to_string()), + ); + if has_flat_model { + enforced.data.insert( + "model".to_string(), + serde_json::Value::String(model.to_string()), + ); + } + } else { + enforced.data.insert( + "model".to_string(), + serde_json::Value::String(model.to_string()), + ); + } + enforced +} + +#[cfg(test)] +mod tests { + use super::*; + + fn event(value: serde_json::Value) -> ResponsesWsEvent { + serde_json::from_value(value).expect("valid event") + } + + #[test] + fn url_construction_matches_python_defaults_and_query_behavior() { + assert_eq!( + complete_websocket_url(None, "gpt-5", true), + "wss://api.openai.com/v1/responses?model=gpt-5" + ); + assert_eq!( + complete_websocket_url(Some("http://localhost:8080/"), "gpt 5", true), + "ws://localhost:8080/responses?model=gpt%205" + ); + assert_eq!( + complete_websocket_url(Some("https://example.test/v1?foo=bar"), "gpt-5", true), + "wss://example.test/v1/responses?foo=bar&model=gpt-5" + ); + assert_eq!( + complete_websocket_url(Some("https://example.test?model=existing"), "gpt-5", true), + "wss://example.test/responses?model=existing" + ); + } + + #[test] + fn enforce_model_overrides_flat_and_nested_values() { + let flat = enforce_model( + &event(serde_json::json!({"type":"response.create","model":"wrong"})), + "gpt-5", + ); + assert_eq!(flat.model(), Some("gpt-5")); + let nested = enforce_model( + &event(serde_json::json!({ + "type":"response.create", + "model":"wrong", + "response":{"model":"also-wrong"} + })), + "gpt-5", + ); + assert_eq!(nested.model(), Some("gpt-5")); + assert_eq!( + nested + .data + .get("response") + .and_then(|value| value.get("model")), + Some(&serde_json::json!("gpt-5")) + ); + let nested_without_flat = enforce_model( + &event(serde_json::json!({ + "type":"response.create", + "response":{"model":"also-wrong"} + })), + "gpt-5", + ); + assert!(!nested_without_flat.data.contains_key("model")); + } +} diff --git a/litellm-rust/crates/providers/src/bedrock/audio_transcription/mod.rs b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/bedrock/audio_transcription/mod.rs rename to litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs diff --git a/litellm-rust/crates/providers/src/bedrock/chat/converse_transformation.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/bedrock/chat/converse_transformation.rs rename to litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs diff --git a/litellm-rust/crates/providers/src/bedrock/chat/mod.rs b/litellm-rust/crates/llms/src/bedrock/chat/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/bedrock/chat/mod.rs rename to litellm-rust/crates/llms/src/bedrock/chat/mod.rs diff --git a/litellm-rust/crates/providers/src/bedrock/chat/tests.rs b/litellm-rust/crates/llms/src/bedrock/chat/tests.rs similarity index 100% rename from litellm-rust/crates/providers/src/bedrock/chat/tests.rs rename to litellm-rust/crates/llms/src/bedrock/chat/tests.rs diff --git a/litellm-rust/crates/providers/src/bedrock/mod.rs b/litellm-rust/crates/llms/src/bedrock/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/bedrock/mod.rs rename to litellm-rust/crates/llms/src/bedrock/mod.rs diff --git a/litellm-rust/crates/providers/src/lib.rs b/litellm-rust/crates/llms/src/lib.rs similarity index 81% rename from litellm-rust/crates/providers/src/lib.rs rename to litellm-rust/crates/llms/src/lib.rs index 67e65d0b8b4..f8e611b53ad 100644 --- a/litellm-rust/crates/providers/src/lib.rs +++ b/litellm-rust/crates/llms/src/lib.rs @@ -2,3 +2,4 @@ pub mod anthropic; pub mod azure_ai; pub mod base_llm; pub mod bedrock; +pub mod openai; diff --git a/litellm-rust/crates/core/src/llms/openai/mod.rs b/litellm-rust/crates/llms/src/openai/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/llms/openai/mod.rs rename to litellm-rust/crates/llms/src/openai/mod.rs diff --git a/litellm-rust/crates/llms/src/openai/responses/mod.rs b/litellm-rust/crates/llms/src/openai/responses/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/openai/responses/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/core/src/llms/openai/responses/transformation.rs b/litellm-rust/crates/llms/src/openai/responses/transformation.rs similarity index 91% rename from litellm-rust/crates/core/src/llms/openai/responses/transformation.rs rename to litellm-rust/crates/llms/src/openai/responses/transformation.rs index ddd160af7a4..f01ec4ad146 100644 --- a/litellm-rust/crates/core/src/llms/openai/responses/transformation.rs +++ b/litellm-rust/crates/llms/src/openai/responses/transformation.rs @@ -1,8 +1,8 @@ use litellm_types::responses::streaming_websocket::{ResponsesWsEvent, ResponsesWsTransformResult}; -use crate::responses::{ - Error, - websocket::{ResponsesWebSocketProviderConfig, enforce_model}, +use crate::base_llm::{ + chat::transformation::Error, + responses::transformation::{ResponsesWebSocketProviderConfig, enforce_model}, }; pub struct OpenAiResponsesApiConfig; diff --git a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/mod.rs b/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/mod.rs deleted file mode 100644 index ba63992f3cb..00000000000 --- a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod messages;