From 49c50739d7ac6900ed8a1382f5afe1cf146dd1f9 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Thu, 17 Sep 2026 22:04:16 -0700 Subject: [PATCH] refactor(rust): rename providers to llms and move non-OCR provider code out of core litellm-providers becomes litellm-llms, mirroring litellm/llms. The Anthropic batches, count_tokens, Messages stream iterator and chat stream handler, the OpenAI Responses websocket config and its base trait (with URL and model helpers), and the StreamTransformer base iterator now live at their Python paths in that crate. Anthropic stream decode errors move with the iterator, and core drops its duplicate OAuth prefix constant and the unused framing dependency. Co-Authored-By: Claude Opus 5 --- litellm-rust/Cargo.lock | 16 +- litellm-rust/Cargo.toml | 2 +- litellm-rust/crates/core/Cargo.toml | 5 +- .../core/src/audio_transcription/error.rs | 2 +- .../core/src/audio_transcription/handler.rs | 2 +- .../core/src/audio_transcription/prepare.rs | 2 +- .../core/src/audio_transcription/types.rs | 2 +- .../core/src/chat_completions/common_utils.rs | 4 +- .../crates/core/src/chat_completions/error.rs | 2 +- .../core/src/chat_completions/handler.rs | 4 +- .../crates/core/src/chat_completions/mod.rs | 1 - .../core/src/chat_completions/prepare.rs | 2 +- .../crates/core/src/chat_completions/tests.rs | 2 +- .../crates/core/src/chat_completions/types.rs | 2 +- litellm-rust/crates/core/src/constants.rs | 7 - .../core/src/llms/anthropic/chat/mod.rs | 1 - .../experimental_pass_through/messages/mod.rs | 3 - .../crates/core/src/llms/anthropic/mod.rs | 2 - litellm-rust/crates/core/src/llms/mod.rs | 2 - .../crates/core/src/messages/common_utils.rs | 2 +- .../crates/core/src/messages/error.rs | 22 +-- .../crates/core/src/messages/prepare.rs | 2 +- .../crates/core/src/messages/types.rs | 2 +- .../crates/core/src/responses/websocket.rs | 179 +---------------- .../crates/{providers => llms}/Cargo.toml | 11 +- .../src/anthropic/batches}/mod.rs | 0 .../src/anthropic/batches/transformation.rs} | 6 +- .../src/anthropic/chat/handler.rs} | 10 +- .../crates/llms/src/anthropic/chat/mod.rs | 2 + .../src/anthropic/chat/tests.rs | 0 .../src/anthropic/chat/transformation.rs | 0 .../src/anthropic/count_tokens}/mod.rs | 0 .../anthropic/count_tokens/transformation.rs} | 2 +- .../experimental_pass_through/messages/mod.rs | 2 + .../messages/streaming_iterator.rs} | 14 +- .../messages/transformation.rs | 0 .../experimental_pass_through/mod.rs | 0 .../{providers => llms}/src/anthropic/mod.rs | 2 + .../anthropic/messages_transformation.rs | 0 .../src/azure_ai/anthropic/mod.rs | 0 .../{providers => llms}/src/azure_ai/mod.rs | 0 .../src/base_llm/anthropic_messages}/mod.rs | 0 .../anthropic_messages/transformation.rs | 0 .../src/base_llm/audio_transcription}/mod.rs | 0 .../audio_transcription/transformation.rs | 0 .../src/base_llm/base_model_iterator.rs} | 0 .../src/base_llm/chat}/mod.rs | 0 .../src/base_llm/chat/transformation.rs | 0 .../{providers => llms}/src/base_llm/mod.rs | 2 + .../src/base_llm/responses}/mod.rs | 0 .../src/base_llm/responses/transformation.rs | 180 ++++++++++++++++++ .../src/bedrock/audio_transcription/mod.rs | 0 .../bedrock/chat/converse_transformation.rs | 0 .../src/bedrock/chat/mod.rs | 0 .../src/bedrock/chat/tests.rs | 0 .../{providers => llms}/src/bedrock/mod.rs | 0 .../crates/{providers => llms}/src/lib.rs | 1 + .../{core/src/llms => llms/src}/openai/mod.rs | 0 .../crates/llms/src/openai/responses/mod.rs | 1 + .../src}/openai/responses/transformation.rs | 6 +- .../experimental_pass_through/mod.rs | 1 - 61 files changed, 257 insertions(+), 253 deletions(-) delete mode 100644 litellm-rust/crates/core/src/llms/anthropic/chat/mod.rs delete mode 100644 litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/mod.rs delete mode 100644 litellm-rust/crates/core/src/llms/anthropic/mod.rs rename litellm-rust/crates/{providers => llms}/Cargo.toml (59%) rename litellm-rust/crates/{core/src/llms/openai/responses => llms/src/anthropic/batches}/mod.rs (100%) rename litellm-rust/crates/{core/src/llms/anthropic/experimental_pass_through/messages/batches.rs => llms/src/anthropic/batches/transformation.rs} (98%) rename litellm-rust/crates/{core/src/llms/anthropic/chat/streaming.rs => llms/src/anthropic/chat/handler.rs} (93%) create mode 100644 litellm-rust/crates/llms/src/anthropic/chat/mod.rs rename litellm-rust/crates/{providers => llms}/src/anthropic/chat/tests.rs (100%) rename litellm-rust/crates/{providers => llms}/src/anthropic/chat/transformation.rs (100%) rename litellm-rust/crates/{providers/src/anthropic/chat => llms/src/anthropic/count_tokens}/mod.rs (100%) rename litellm-rust/crates/{core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs => llms/src/anthropic/count_tokens/transformation.rs} (98%) create mode 100644 litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs rename litellm-rust/crates/{core/src/llms/anthropic/experimental_pass_through/messages/streaming.rs => llms/src/anthropic/experimental_pass_through/messages/streaming_iterator.rs} (94%) rename litellm-rust/crates/{providers => llms}/src/anthropic/experimental_pass_through/messages/transformation.rs (100%) rename litellm-rust/crates/{core/src/llms => llms/src}/anthropic/experimental_pass_through/mod.rs (100%) rename litellm-rust/crates/{providers => llms}/src/anthropic/mod.rs (74%) rename litellm-rust/crates/{providers => llms}/src/azure_ai/anthropic/messages_transformation.rs (100%) rename litellm-rust/crates/{providers => llms}/src/azure_ai/anthropic/mod.rs (100%) rename litellm-rust/crates/{providers => llms}/src/azure_ai/mod.rs (100%) rename litellm-rust/crates/{providers/src/anthropic/experimental_pass_through/messages => llms/src/base_llm/anthropic_messages}/mod.rs (100%) rename litellm-rust/crates/{providers => llms}/src/base_llm/anthropic_messages/transformation.rs (100%) rename litellm-rust/crates/{providers/src/base_llm/anthropic_messages => llms/src/base_llm/audio_transcription}/mod.rs (100%) rename litellm-rust/crates/{providers => llms}/src/base_llm/audio_transcription/transformation.rs (100%) rename litellm-rust/crates/{core/src/chat_completions/streaming.rs => llms/src/base_llm/base_model_iterator.rs} (100%) rename litellm-rust/crates/{providers/src/base_llm/audio_transcription => llms/src/base_llm/chat}/mod.rs (100%) rename litellm-rust/crates/{providers => llms}/src/base_llm/chat/transformation.rs (100%) rename litellm-rust/crates/{providers => llms}/src/base_llm/mod.rs (59%) rename litellm-rust/crates/{providers/src/base_llm/chat => llms/src/base_llm/responses}/mod.rs (100%) create mode 100644 litellm-rust/crates/llms/src/base_llm/responses/transformation.rs rename litellm-rust/crates/{providers => llms}/src/bedrock/audio_transcription/mod.rs (100%) rename litellm-rust/crates/{providers => llms}/src/bedrock/chat/converse_transformation.rs (100%) rename litellm-rust/crates/{providers => llms}/src/bedrock/chat/mod.rs (100%) rename litellm-rust/crates/{providers => llms}/src/bedrock/chat/tests.rs (100%) rename litellm-rust/crates/{providers => llms}/src/bedrock/mod.rs (100%) rename litellm-rust/crates/{providers => llms}/src/lib.rs (81%) rename litellm-rust/crates/{core/src/llms => llms/src}/openai/mod.rs (100%) create mode 100644 litellm-rust/crates/llms/src/openai/responses/mod.rs rename litellm-rust/crates/{core/src/llms => llms/src}/openai/responses/transformation.rs (91%) delete mode 100644 litellm-rust/crates/providers/src/anthropic/experimental_pass_through/mod.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 196f282b5fb..075ee076658 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2025,8 +2025,6 @@ dependencies = [ name = "litellm-core" version = "0.1.0" dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-types", "base64 0.22.1", "bytes", "data-url", @@ -2037,8 +2035,7 @@ dependencies = [ "litellm-auth-gcp", "litellm-callbacks", "litellm-core-utils", - "litellm-framing", - "litellm-providers", + "litellm-llms", "litellm-types", "mime_guess", "moka", @@ -2107,17 +2104,26 @@ dependencies = [ ] [[package]] -name = "litellm-providers" +name = "litellm-llms" version = "0.1.0" dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-types", + "base64 0.22.1", + "bytes", + "futures-util", "litellm-auth", "litellm-auth-aws", "litellm-core-utils", + "litellm-framing", "litellm-types", "rstest", "serde", "serde_json", "thiserror 2.0.19", + "time", + "tokio", + "url", ] [[package]] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 89b217cadda..f8377138050 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -17,7 +17,7 @@ litellm-auth = { path = "crates/auth" } litellm-auth-aws = { path = "crates/auth-aws" } litellm-auth-azure = { path = "crates/auth-azure" } litellm-auth-gcp = { path = "crates/auth-gcp" } -litellm-providers = { path = "crates/providers" } +litellm-llms = { path = "crates/llms" } litellm-types = { path = "crates/types" } litellm-core-utils = { path = "crates/core-utils" } litellm-cache = { path = "crates/cache" } diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index 33f44d66bb2..7695e257bcd 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -18,8 +18,7 @@ litellm-auth.workspace = true litellm-auth-aws.workspace = true litellm-auth-azure.workspace = true litellm-auth-gcp.workspace = true -litellm-providers.workspace = true -litellm-framing.workspace = true +litellm-llms.workspace = true moka.workspace = true mime_guess = "2.0.5" rand.workspace = true @@ -41,7 +40,5 @@ url.workspace = true veil.workspace = true [dev-dependencies] -aws-smithy-eventstream = "=0.61.1" -aws-smithy-types = "1.6.1" rstest.workspace = true rstest_reuse.workspace = true diff --git a/litellm-rust/crates/core/src/audio_transcription/error.rs b/litellm-rust/crates/core/src/audio_transcription/error.rs index cfa9e4cc5e2..6c08727795b 100644 --- a/litellm-rust/crates/core/src/audio_transcription/error.rs +++ b/litellm-rust/crates/core/src/audio_transcription/error.rs @@ -1,4 +1,4 @@ -use litellm_providers::base_llm::chat::transformation::Error as LlmError; +use litellm_llms::base_llm::chat::transformation::Error as LlmError; #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { diff --git a/litellm-rust/crates/core/src/audio_transcription/handler.rs b/litellm-rust/crates/core/src/audio_transcription/handler.rs index e79833cf201..01cf236e118 100644 --- a/litellm-rust/crates/core/src/audio_transcription/handler.rs +++ b/litellm-rust/crates/core/src/audio_transcription/handler.rs @@ -48,7 +48,7 @@ async fn signed_headers( use std::{collections::BTreeMap, time::SystemTime}; use litellm_auth_aws::{aws_auth_config, resolve_credentials, sign_bedrock_post}; - use litellm_providers::base_llm::audio_transcription::transformation::AudioTranscriptionAuth; + use litellm_llms::base_llm::audio_transcription::transformation::AudioTranscriptionAuth; let AudioTranscriptionAuth::AwsSigV4 { region, .. } = &request.auth else { return Ok(request.upstream_headers.clone()); diff --git a/litellm-rust/crates/core/src/audio_transcription/prepare.rs b/litellm-rust/crates/core/src/audio_transcription/prepare.rs index 0e230117834..25c9afd5ea3 100644 --- a/litellm-rust/crates/core/src/audio_transcription/prepare.rs +++ b/litellm-rust/crates/core/src/audio_transcription/prepare.rs @@ -1,5 +1,5 @@ use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; -use litellm_providers::{ +use litellm_llms::{ base_llm::audio_transcription::transformation::{ AudioTranscriptionAuth, BaseAudioTranscriptionConfig, }, diff --git a/litellm-rust/crates/core/src/audio_transcription/types.rs b/litellm-rust/crates/core/src/audio_transcription/types.rs index 7878e4bd175..ca09dd945be 100644 --- a/litellm-rust/crates/core/src/audio_transcription/types.rs +++ b/litellm-rust/crates/core/src/audio_transcription/types.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use litellm_providers::base_llm::audio_transcription::transformation::{ +use litellm_llms::base_llm::audio_transcription::transformation::{ AudioTranscriptionAuth, BaseAudioTranscriptionConfig, }; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/core/src/chat_completions/common_utils.rs b/litellm-rust/crates/core/src/chat_completions/common_utils.rs index 309cc781cc0..bb524db661e 100644 --- a/litellm-rust/crates/core/src/chat_completions/common_utils.rs +++ b/litellm-rust/crates/core/src/chat_completions/common_utils.rs @@ -1,4 +1,4 @@ -use litellm_providers::{ +use litellm_llms::{ anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG, base_llm::chat::transformation::BaseConfig, }; @@ -13,7 +13,7 @@ pub(super) fn chat_completions_provider_config(provider: &str) -> Option<&'stati match provider { "anthropic" => Some(&ANTHROPIC_CHAT_COMPLETIONS_CONFIG), "bedrock" => Some( - &litellm_providers::bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, + &litellm_llms::bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, ), _ => None, } diff --git a/litellm-rust/crates/core/src/chat_completions/error.rs b/litellm-rust/crates/core/src/chat_completions/error.rs index cfa9e4cc5e2..6c08727795b 100644 --- a/litellm-rust/crates/core/src/chat_completions/error.rs +++ b/litellm-rust/crates/core/src/chat_completions/error.rs @@ -1,4 +1,4 @@ -use litellm_providers::base_llm::chat::transformation::Error as LlmError; +use litellm_llms::base_llm::chat::transformation::Error as LlmError; #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index ae8c0dd6144..664841318fc 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -1,6 +1,4 @@ -use litellm_providers::base_llm::chat::transformation::{ - ChatCompletionsAuth, ProviderChatResponseData, -}; +use litellm_llms::base_llm::chat::transformation::{ChatCompletionsAuth, ProviderChatResponseData}; use litellm_types::utils::ChatCompletionsResponse; use serde_json::Value; diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index 5c5cc48692a..81d35044d08 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -13,7 +13,6 @@ mod client; mod common_utils; pub(crate) mod handler; mod prepare; -pub mod streaming; use handler::execute_chat_completions_provider_call; use litellm_types::utils::ChatCompletionsResponse; use prepare::{parse_messages, resolve_provider_config, resolve_request}; diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/core/src/chat_completions/prepare.rs index 882da4e584b..4dae9af966c 100644 --- a/litellm-rust/crates/core/src/chat_completions/prepare.rs +++ b/litellm-rust/crates/core/src/chat_completions/prepare.rs @@ -1,5 +1,5 @@ use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; -use litellm_providers::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; +use litellm_llms::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; use litellm_types::llms::openai::ChatMessage; use serde_json::Value; diff --git a/litellm-rust/crates/core/src/chat_completions/tests.rs b/litellm-rust/crates/core/src/chat_completions/tests.rs index b8bf8625e9c..b52cc538a65 100644 --- a/litellm-rust/crates/core/src/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/chat_completions/tests.rs @@ -1,4 +1,4 @@ -use litellm_providers::base_llm::chat::transformation::ChatCompletionsAuth; +use litellm_llms::base_llm::chat::transformation::ChatCompletionsAuth; use serde_json::{Map, Value, json}; use super::{ diff --git a/litellm-rust/crates/core/src/chat_completions/types.rs b/litellm-rust/crates/core/src/chat_completions/types.rs index 1687ef0a792..882611d5862 100644 --- a/litellm-rust/crates/core/src/chat_completions/types.rs +++ b/litellm-rust/crates/core/src/chat_completions/types.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use litellm_providers::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; +use litellm_llms::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; use litellm_types::llms::openai::ChatMessage; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index 4ff4333c4ac..607ce32c6d4 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -1,6 +1,4 @@ pub const OPENAI_DEFAULT_API_BASE: &str = "https://api.openai.com"; -pub const OPENAI_RESPONSES_DEFAULT_API_BASE: &str = "https://api.openai.com/v1"; -pub const OPENAI_RESPONSES_PATH: &str = "/responses"; /// Full-request timeout ceiling for Anthropic Messages provider calls, in /// seconds. Mirrors the Python Anthropic Messages default. The per-request @@ -18,11 +16,6 @@ pub(crate) const UPSTREAM_ERROR_BODY_MAX_CHARS: usize = 256; /// does not carry an explicit provider prefix. pub const ANTHROPIC_MESSAGES_PROVIDER: &str = "anthropic"; -/// Prefix identifying an Anthropic OAuth token. Mirrors Python's -/// `ANTHROPIC_OAUTH_TOKEN_PREFIX`, which is what makes `validate_environment` -/// authenticate with `authorization` and drop `x-api-key` entirely. -pub(crate) const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat"; - /// Full-request timeout ceiling for chat completions provider calls, in /// seconds. Mirrors the Python chat completions default. pub(crate) const CHAT_COMPLETIONS_TIMEOUT_SECS: u64 = 600; diff --git a/litellm-rust/crates/core/src/llms/anthropic/chat/mod.rs b/litellm-rust/crates/core/src/llms/anthropic/chat/mod.rs deleted file mode 100644 index 7bf4fc46291..00000000000 --- a/litellm-rust/crates/core/src/llms/anthropic/chat/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod streaming; diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/mod.rs b/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/mod.rs deleted file mode 100644 index 42d4fcdde0f..00000000000 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod batches; -pub mod count_tokens; -pub mod streaming; diff --git a/litellm-rust/crates/core/src/llms/anthropic/mod.rs b/litellm-rust/crates/core/src/llms/anthropic/mod.rs deleted file mode 100644 index 4943d80a45c..00000000000 --- a/litellm-rust/crates/core/src/llms/anthropic/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod chat; -pub mod experimental_pass_through; diff --git a/litellm-rust/crates/core/src/llms/mod.rs b/litellm-rust/crates/core/src/llms/mod.rs index 4b93a5f971c..8b0a67f6a94 100644 --- a/litellm-rust/crates/core/src/llms/mod.rs +++ b/litellm-rust/crates/core/src/llms/mod.rs @@ -1,8 +1,6 @@ -pub mod anthropic; pub mod azure_ai; pub mod base_llm; pub(crate) mod cohere; pub(crate) mod mistral; -pub mod openai; pub(crate) mod reducto; pub(crate) mod vertex_ai; diff --git a/litellm-rust/crates/core/src/messages/common_utils.rs b/litellm-rust/crates/core/src/messages/common_utils.rs index 81d67520abe..f73f62c471c 100644 --- a/litellm-rust/crates/core/src/messages/common_utils.rs +++ b/litellm-rust/crates/core/src/messages/common_utils.rs @@ -1,4 +1,4 @@ -use litellm_providers::{ +use litellm_llms::{ anthropic::experimental_pass_through::messages::transformation::ANTHROPIC_MESSAGES_CONFIG, azure_ai::anthropic::messages_transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG, base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig, diff --git a/litellm-rust/crates/core/src/messages/error.rs b/litellm-rust/crates/core/src/messages/error.rs index 7a9c1c1d20a..58a88b6d16e 100644 --- a/litellm-rust/crates/core/src/messages/error.rs +++ b/litellm-rust/crates/core/src/messages/error.rs @@ -1,4 +1,4 @@ -use litellm_providers::base_llm::chat::transformation::Error as LlmError; +use litellm_llms::base_llm::chat::transformation::Error as LlmError; #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { @@ -18,16 +18,6 @@ pub enum Error { Transport(#[from] crate::transport::Error), #[error(transparent)] Headers(#[from] crate::http_utils::HeaderError), - #[error("stream framing failed: {0}")] - StreamFraming(String), - #[error("Anthropic SSE frame has no data")] - MissingStreamData, - #[error("Anthropic stream event is invalid: {0}")] - InvalidStreamEvent(String), - #[error("Bedrock event payload is invalid: {0}")] - InvalidBedrockPayload(String), - #[error("Bedrock event payload has invalid base64: {0}")] - InvalidBedrockBase64(String), } impl From for Error { @@ -57,14 +47,6 @@ impl Error { } pub fn is_response(&self) -> bool { - matches!( - self, - Self::InvalidResponse(_) - | Self::StreamFraming(_) - | Self::MissingStreamData - | Self::InvalidStreamEvent(_) - | Self::InvalidBedrockPayload(_) - | Self::InvalidBedrockBase64(_) - ) + matches!(self, Self::InvalidResponse(_)) } } diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index d7e996d74e8..8b676803871 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,5 +1,5 @@ use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; -use litellm_providers::base_llm::anthropic_messages::transformation::{ +use litellm_llms::base_llm::anthropic_messages::transformation::{ BaseAnthropicMessagesConfig, MessagesAuthStrategy, }; use serde_json::{Map, Value}; diff --git a/litellm-rust/crates/core/src/messages/types.rs b/litellm-rust/crates/core/src/messages/types.rs index 928eca222be..a73ceffad7a 100644 --- a/litellm-rust/crates/core/src/messages/types.rs +++ b/litellm-rust/crates/core/src/messages/types.rs @@ -1,6 +1,6 @@ use std::time::Duration; -use litellm_providers::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig; +use litellm_llms::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig; use serde_json::{Map, Value}; pub struct MessagesRequest<'a> { diff --git a/litellm-rust/crates/core/src/responses/websocket.rs b/litellm-rust/crates/core/src/responses/websocket.rs index a91fbcf272f..b601af74d55 100644 --- a/litellm-rust/crates/core/src/responses/websocket.rs +++ b/litellm-rust/crates/core/src/responses/websocket.rs @@ -6,9 +6,7 @@ use std::{ }; use futures_util::{SinkExt, StreamExt}; -use litellm_types::responses::streaming_websocket::{ - ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult, -}; +use litellm_types::responses::streaming_websocket::ResponsesWsEventType; use rustls::{ClientConfig, RootCertStore}; use tokio::{net::TcpStream, sync::Mutex}; use tokio_tungstenite::{ @@ -23,119 +21,6 @@ use tokio_tungstenite::{ }; use super::Error; -use crate::constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH}; - -pub trait ResponsesWebSocketProviderConfig: Sync { - fn supports_native_websocket(&self) -> bool { - false - } - - fn model_in_websocket_url(&self) -> bool { - true - } - - fn complete_websocket_url(&self, api_base: Option<&str>, model: &str) -> String { - complete_websocket_url(api_base, model, self.model_in_websocket_url()) - } - - fn transform_ws_request( - &self, - event: &ResponsesWsEvent, - model: &str, - ) -> Result; - - fn transform_ws_response( - &self, - event: &ResponsesWsEvent, - model: &str, - ) -> Result; -} - -pub fn complete_websocket_url( - api_base: Option<&str>, - model: &str, - model_in_websocket_url: bool, -) -> String { - let base = api_base - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or(OPENAI_RESPONSES_DEFAULT_API_BASE); - let (base_without_query, query) = base - .split_once('?') - .map_or((base, None), |(value, query)| (value, Some(query))); - let response_url = format!( - "{}{}", - base_without_query.trim_end_matches('/'), - OPENAI_RESPONSES_PATH - ); - let scheme_flipped = if let Some(rest) = response_url.strip_prefix("https://") { - format!("wss://{rest}") - } else if let Some(rest) = response_url.strip_prefix("http://") { - format!("ws://{rest}") - } else { - response_url - }; - let url = query.map_or(scheme_flipped.clone(), |value| { - format!("{scheme_flipped}?{value}") - }); - if !model_in_websocket_url - || query.is_some_and(|value| { - value - .split('&') - .any(|part| part.split('=').next() == Some("model")) - }) - { - return url; - } - format!( - "{url}{}model={}", - if query.is_some() { "&" } else { "?" }, - percent_encode(model) - ) -} - -fn percent_encode(value: &str) -> String { - value - .bytes() - .map(|byte| { - if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { - format!("{}", byte as char) - } else { - format!("%{byte:02X}") - } - }) - .collect() -} - -pub fn enforce_model(event: &ResponsesWsEvent, model: &str) -> ResponsesWsEvent { - if !event.is_response_create() { - return event.clone(); - } - let mut enforced = event.clone(); - let has_flat_model = enforced.data.contains_key("model"); - if let Some(response) = enforced - .data - .get_mut("response") - .and_then(serde_json::Value::as_object_mut) - { - response.insert( - "model".to_string(), - serde_json::Value::String(model.to_string()), - ); - if has_flat_model { - enforced.data.insert( - "model".to_string(), - serde_json::Value::String(model.to_string()), - ); - } - } else { - enforced.data.insert( - "model".to_string(), - serde_json::Value::String(model.to_string()), - ); - } - enforced -} pub fn is_terminal_event(event_type: &ResponsesWsEventType) -> bool { matches!( @@ -285,65 +170,3 @@ impl ResponsesWebSocketConnection { Ok(()) } } - -#[cfg(test)] -mod tests { - use super::*; - - fn event(value: serde_json::Value) -> ResponsesWsEvent { - serde_json::from_value(value).expect("valid event") - } - - #[test] - fn url_construction_matches_python_defaults_and_query_behavior() { - assert_eq!( - complete_websocket_url(None, "gpt-5", true), - "wss://api.openai.com/v1/responses?model=gpt-5" - ); - assert_eq!( - complete_websocket_url(Some("http://localhost:8080/"), "gpt 5", true), - "ws://localhost:8080/responses?model=gpt%205" - ); - assert_eq!( - complete_websocket_url(Some("https://example.test/v1?foo=bar"), "gpt-5", true), - "wss://example.test/v1/responses?foo=bar&model=gpt-5" - ); - assert_eq!( - complete_websocket_url(Some("https://example.test?model=existing"), "gpt-5", true), - "wss://example.test/responses?model=existing" - ); - } - - #[test] - fn enforce_model_overrides_flat_and_nested_values() { - let flat = enforce_model( - &event(serde_json::json!({"type":"response.create","model":"wrong"})), - "gpt-5", - ); - assert_eq!(flat.model(), Some("gpt-5")); - let nested = enforce_model( - &event(serde_json::json!({ - "type":"response.create", - "model":"wrong", - "response":{"model":"also-wrong"} - })), - "gpt-5", - ); - assert_eq!(nested.model(), Some("gpt-5")); - assert_eq!( - nested - .data - .get("response") - .and_then(|value| value.get("model")), - Some(&serde_json::json!("gpt-5")) - ); - let nested_without_flat = enforce_model( - &event(serde_json::json!({ - "type":"response.create", - "response":{"model":"also-wrong"} - })), - "gpt-5", - ); - assert!(!nested_without_flat.data.contains_key("model")); - } -} diff --git a/litellm-rust/crates/providers/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml similarity index 59% rename from litellm-rust/crates/providers/Cargo.toml rename to litellm-rust/crates/llms/Cargo.toml index f137e2ddbe4..c7564044fcb 100644 --- a/litellm-rust/crates/providers/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "litellm-providers" +name = "litellm-llms" version = "0.1.0" edition.workspace = true license.workspace = true @@ -10,9 +10,18 @@ litellm-types.workspace = true litellm-core-utils.workspace = true litellm-auth.workspace = true litellm-auth-aws.workspace = true +litellm-framing.workspace = true +base64.workspace = true +bytes.workspace = true +futures-util.workspace = true serde.workspace = true serde_json.workspace = true thiserror.workspace = true +time.workspace = true +url.workspace = true [dev-dependencies] +aws-smithy-eventstream = "=0.61.1" +aws-smithy-types = "1.6.1" rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/core/src/llms/openai/responses/mod.rs b/litellm-rust/crates/llms/src/anthropic/batches/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/llms/openai/responses/mod.rs rename to litellm-rust/crates/llms/src/anthropic/batches/mod.rs diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs b/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs similarity index 98% rename from litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs rename to litellm-rust/crates/llms/src/anthropic/batches/transformation.rs index e371a24c057..94e4dc7838a 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs +++ b/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs @@ -1,11 +1,13 @@ -use litellm_providers::anthropic::experimental_pass_through::messages::transformation::resolve_anthropic_api_base; use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; use serde::{Deserialize, Serialize}; use serde_json::Value; use time::OffsetDateTime; use url::Url; -use crate::messages::Error; +use crate::{ + anthropic::experimental_pass_through::messages::transformation::resolve_anthropic_api_base, + base_llm::chat::transformation::Error, +}; const BATCHES_PATH_SUFFIX: &str = "/v1/messages/batches"; diff --git a/litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs b/litellm-rust/crates/llms/src/anthropic/chat/handler.rs similarity index 93% rename from litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs rename to litellm-rust/crates/llms/src/anthropic/chat/handler.rs index 86c6e010e84..a80cfbf28bd 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/handler.rs @@ -6,11 +6,13 @@ use litellm_types::{ }; use serde_json::Value; -use super::super::experimental_pass_through::messages::streaming::{ - AnthropicContentBlock, AnthropicContentBlockDelta, AnthropicMessagesStreamEvent, - AnthropicStreamUsage, +use crate::{ + anthropic::experimental_pass_through::messages::streaming_iterator::{ + AnthropicContentBlock, AnthropicContentBlockDelta, AnthropicMessagesStreamEvent, + AnthropicStreamUsage, + }, + base_llm::{base_model_iterator::StreamTransformer, chat::transformation::Error}, }; -use crate::chat_completions::{Error, streaming::StreamTransformer}; #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum AnthropicJsonChunkType { diff --git a/litellm-rust/crates/llms/src/anthropic/chat/mod.rs b/litellm-rust/crates/llms/src/anthropic/chat/mod.rs new file mode 100644 index 00000000000..f0050b7dc71 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/chat/mod.rs @@ -0,0 +1,2 @@ +pub mod handler; +pub mod transformation; diff --git a/litellm-rust/crates/providers/src/anthropic/chat/tests.rs b/litellm-rust/crates/llms/src/anthropic/chat/tests.rs similarity index 100% rename from litellm-rust/crates/providers/src/anthropic/chat/tests.rs rename to litellm-rust/crates/llms/src/anthropic/chat/tests.rs diff --git a/litellm-rust/crates/providers/src/anthropic/chat/transformation.rs b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/anthropic/chat/transformation.rs rename to litellm-rust/crates/llms/src/anthropic/chat/transformation.rs diff --git a/litellm-rust/crates/providers/src/anthropic/chat/mod.rs b/litellm-rust/crates/llms/src/anthropic/count_tokens/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/anthropic/chat/mod.rs rename to litellm-rust/crates/llms/src/anthropic/count_tokens/mod.rs diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs b/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs similarity index 98% rename from litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs rename to litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs index 82a2a54c961..a4d8c57ca4f 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs +++ b/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs @@ -2,7 +2,7 @@ use litellm_types::llms::anthropic_messages::anthropic_request::{AnthropicMessag use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::{constants::ANTHROPIC_OAUTH_TOKEN_PREFIX, messages::Error}; +use crate::{anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX, base_llm::chat::transformation::Error}; const COUNT_TOKENS_ENDPOINT: &str = "https://api.anthropic.com/v1/messages/count_tokens"; const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01"; diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs new file mode 100644 index 00000000000..481d98c4e9d --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs @@ -0,0 +1,2 @@ +pub mod streaming_iterator; +pub mod transformation; diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/streaming.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/streaming_iterator.rs similarity index 94% rename from litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/streaming.rs rename to litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/streaming_iterator.rs index 92a36265df7..35e7d5820b0 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/streaming.rs +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/streaming_iterator.rs @@ -9,7 +9,19 @@ use litellm_framing::{ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use crate::messages::Error; +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("stream framing failed: {0}")] + StreamFraming(String), + #[error("Anthropic SSE frame has no data")] + MissingStreamData, + #[error("Anthropic stream event is invalid: {0}")] + InvalidStreamEvent(String), + #[error("Bedrock event payload is invalid: {0}")] + InvalidBedrockPayload(String), + #[error("Bedrock event payload has invalid base64: {0}")] + InvalidBedrockBase64(String), +} #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct AnthropicStreamUsage { diff --git a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/transformation.rs rename to litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/transformation.rs diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/mod.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/mod.rs rename to litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs diff --git a/litellm-rust/crates/providers/src/anthropic/mod.rs b/litellm-rust/crates/llms/src/anthropic/mod.rs similarity index 74% rename from litellm-rust/crates/providers/src/anthropic/mod.rs rename to litellm-rust/crates/llms/src/anthropic/mod.rs index 38a59aa6e0d..d181ceaca3c 100644 --- a/litellm-rust/crates/providers/src/anthropic/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/mod.rs @@ -1,4 +1,6 @@ +pub mod batches; pub mod chat; +pub mod count_tokens; pub mod experimental_pass_through; pub const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat"; diff --git a/litellm-rust/crates/providers/src/azure_ai/anthropic/messages_transformation.rs b/litellm-rust/crates/llms/src/azure_ai/anthropic/messages_transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/azure_ai/anthropic/messages_transformation.rs rename to litellm-rust/crates/llms/src/azure_ai/anthropic/messages_transformation.rs diff --git a/litellm-rust/crates/providers/src/azure_ai/anthropic/mod.rs b/litellm-rust/crates/llms/src/azure_ai/anthropic/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/azure_ai/anthropic/mod.rs rename to litellm-rust/crates/llms/src/azure_ai/anthropic/mod.rs diff --git a/litellm-rust/crates/providers/src/azure_ai/mod.rs b/litellm-rust/crates/llms/src/azure_ai/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/azure_ai/mod.rs rename to litellm-rust/crates/llms/src/azure_ai/mod.rs diff --git a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/mod.rs b/litellm-rust/crates/llms/src/base_llm/anthropic_messages/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/mod.rs rename to litellm-rust/crates/llms/src/base_llm/anthropic_messages/mod.rs diff --git a/litellm-rust/crates/providers/src/base_llm/anthropic_messages/transformation.rs b/litellm-rust/crates/llms/src/base_llm/anthropic_messages/transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/anthropic_messages/transformation.rs rename to litellm-rust/crates/llms/src/base_llm/anthropic_messages/transformation.rs diff --git a/litellm-rust/crates/providers/src/base_llm/anthropic_messages/mod.rs b/litellm-rust/crates/llms/src/base_llm/audio_transcription/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/anthropic_messages/mod.rs rename to litellm-rust/crates/llms/src/base_llm/audio_transcription/mod.rs diff --git a/litellm-rust/crates/providers/src/base_llm/audio_transcription/transformation.rs b/litellm-rust/crates/llms/src/base_llm/audio_transcription/transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/audio_transcription/transformation.rs rename to litellm-rust/crates/llms/src/base_llm/audio_transcription/transformation.rs diff --git a/litellm-rust/crates/core/src/chat_completions/streaming.rs b/litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs similarity index 100% rename from litellm-rust/crates/core/src/chat_completions/streaming.rs rename to litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs diff --git a/litellm-rust/crates/providers/src/base_llm/audio_transcription/mod.rs b/litellm-rust/crates/llms/src/base_llm/chat/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/audio_transcription/mod.rs rename to litellm-rust/crates/llms/src/base_llm/chat/mod.rs diff --git a/litellm-rust/crates/providers/src/base_llm/chat/transformation.rs b/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/chat/transformation.rs rename to litellm-rust/crates/llms/src/base_llm/chat/transformation.rs diff --git a/litellm-rust/crates/providers/src/base_llm/mod.rs b/litellm-rust/crates/llms/src/base_llm/mod.rs similarity index 59% rename from litellm-rust/crates/providers/src/base_llm/mod.rs rename to litellm-rust/crates/llms/src/base_llm/mod.rs index b7a1f696440..0c3aa2091a3 100644 --- a/litellm-rust/crates/providers/src/base_llm/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/mod.rs @@ -1,3 +1,5 @@ pub mod anthropic_messages; pub mod audio_transcription; +pub mod base_model_iterator; pub mod chat; +pub mod responses; diff --git a/litellm-rust/crates/providers/src/base_llm/chat/mod.rs b/litellm-rust/crates/llms/src/base_llm/responses/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/chat/mod.rs rename to litellm-rust/crates/llms/src/base_llm/responses/mod.rs diff --git a/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs b/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs new file mode 100644 index 00000000000..0d9cfcfd4cd --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs @@ -0,0 +1,180 @@ +use litellm_types::responses::streaming_websocket::{ResponsesWsEvent, ResponsesWsTransformResult}; + +use crate::base_llm::chat::transformation::Error; + +pub const OPENAI_RESPONSES_DEFAULT_API_BASE: &str = "https://api.openai.com/v1"; +pub const OPENAI_RESPONSES_PATH: &str = "/responses"; + +pub trait ResponsesWebSocketProviderConfig: Sync { + fn supports_native_websocket(&self) -> bool { + false + } + + fn model_in_websocket_url(&self) -> bool { + true + } + + fn complete_websocket_url(&self, api_base: Option<&str>, model: &str) -> String { + complete_websocket_url(api_base, model, self.model_in_websocket_url()) + } + + fn transform_ws_request( + &self, + event: &ResponsesWsEvent, + model: &str, + ) -> Result; + + fn transform_ws_response( + &self, + event: &ResponsesWsEvent, + model: &str, + ) -> Result; +} + +pub fn complete_websocket_url( + api_base: Option<&str>, + model: &str, + model_in_websocket_url: bool, +) -> String { + let base = api_base + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or(OPENAI_RESPONSES_DEFAULT_API_BASE); + let (base_without_query, query) = base + .split_once('?') + .map_or((base, None), |(value, query)| (value, Some(query))); + let response_url = format!( + "{}{}", + base_without_query.trim_end_matches('/'), + OPENAI_RESPONSES_PATH + ); + let scheme_flipped = if let Some(rest) = response_url.strip_prefix("https://") { + format!("wss://{rest}") + } else if let Some(rest) = response_url.strip_prefix("http://") { + format!("ws://{rest}") + } else { + response_url + }; + let url = query.map_or(scheme_flipped.clone(), |value| { + format!("{scheme_flipped}?{value}") + }); + if !model_in_websocket_url + || query.is_some_and(|value| { + value + .split('&') + .any(|part| part.split('=').next() == Some("model")) + }) + { + return url; + } + format!( + "{url}{}model={}", + if query.is_some() { "&" } else { "?" }, + percent_encode(model) + ) +} + +fn percent_encode(value: &str) -> String { + value + .bytes() + .map(|byte| { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + format!("{}", byte as char) + } else { + format!("%{byte:02X}") + } + }) + .collect() +} + +pub fn enforce_model(event: &ResponsesWsEvent, model: &str) -> ResponsesWsEvent { + if !event.is_response_create() { + return event.clone(); + } + let mut enforced = event.clone(); + let has_flat_model = enforced.data.contains_key("model"); + if let Some(response) = enforced + .data + .get_mut("response") + .and_then(serde_json::Value::as_object_mut) + { + response.insert( + "model".to_string(), + serde_json::Value::String(model.to_string()), + ); + if has_flat_model { + enforced.data.insert( + "model".to_string(), + serde_json::Value::String(model.to_string()), + ); + } + } else { + enforced.data.insert( + "model".to_string(), + serde_json::Value::String(model.to_string()), + ); + } + enforced +} + +#[cfg(test)] +mod tests { + use super::*; + + fn event(value: serde_json::Value) -> ResponsesWsEvent { + serde_json::from_value(value).expect("valid event") + } + + #[test] + fn url_construction_matches_python_defaults_and_query_behavior() { + assert_eq!( + complete_websocket_url(None, "gpt-5", true), + "wss://api.openai.com/v1/responses?model=gpt-5" + ); + assert_eq!( + complete_websocket_url(Some("http://localhost:8080/"), "gpt 5", true), + "ws://localhost:8080/responses?model=gpt%205" + ); + assert_eq!( + complete_websocket_url(Some("https://example.test/v1?foo=bar"), "gpt-5", true), + "wss://example.test/v1/responses?foo=bar&model=gpt-5" + ); + assert_eq!( + complete_websocket_url(Some("https://example.test?model=existing"), "gpt-5", true), + "wss://example.test/responses?model=existing" + ); + } + + #[test] + fn enforce_model_overrides_flat_and_nested_values() { + let flat = enforce_model( + &event(serde_json::json!({"type":"response.create","model":"wrong"})), + "gpt-5", + ); + assert_eq!(flat.model(), Some("gpt-5")); + let nested = enforce_model( + &event(serde_json::json!({ + "type":"response.create", + "model":"wrong", + "response":{"model":"also-wrong"} + })), + "gpt-5", + ); + assert_eq!(nested.model(), Some("gpt-5")); + assert_eq!( + nested + .data + .get("response") + .and_then(|value| value.get("model")), + Some(&serde_json::json!("gpt-5")) + ); + let nested_without_flat = enforce_model( + &event(serde_json::json!({ + "type":"response.create", + "response":{"model":"also-wrong"} + })), + "gpt-5", + ); + assert!(!nested_without_flat.data.contains_key("model")); + } +} diff --git a/litellm-rust/crates/providers/src/bedrock/audio_transcription/mod.rs b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/bedrock/audio_transcription/mod.rs rename to litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs diff --git a/litellm-rust/crates/providers/src/bedrock/chat/converse_transformation.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs similarity index 100% rename from litellm-rust/crates/providers/src/bedrock/chat/converse_transformation.rs rename to litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs diff --git a/litellm-rust/crates/providers/src/bedrock/chat/mod.rs b/litellm-rust/crates/llms/src/bedrock/chat/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/bedrock/chat/mod.rs rename to litellm-rust/crates/llms/src/bedrock/chat/mod.rs diff --git a/litellm-rust/crates/providers/src/bedrock/chat/tests.rs b/litellm-rust/crates/llms/src/bedrock/chat/tests.rs similarity index 100% rename from litellm-rust/crates/providers/src/bedrock/chat/tests.rs rename to litellm-rust/crates/llms/src/bedrock/chat/tests.rs diff --git a/litellm-rust/crates/providers/src/bedrock/mod.rs b/litellm-rust/crates/llms/src/bedrock/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/bedrock/mod.rs rename to litellm-rust/crates/llms/src/bedrock/mod.rs diff --git a/litellm-rust/crates/providers/src/lib.rs b/litellm-rust/crates/llms/src/lib.rs similarity index 81% rename from litellm-rust/crates/providers/src/lib.rs rename to litellm-rust/crates/llms/src/lib.rs index 67e65d0b8b4..f8e611b53ad 100644 --- a/litellm-rust/crates/providers/src/lib.rs +++ b/litellm-rust/crates/llms/src/lib.rs @@ -2,3 +2,4 @@ pub mod anthropic; pub mod azure_ai; pub mod base_llm; pub mod bedrock; +pub mod openai; diff --git a/litellm-rust/crates/core/src/llms/openai/mod.rs b/litellm-rust/crates/llms/src/openai/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/llms/openai/mod.rs rename to litellm-rust/crates/llms/src/openai/mod.rs diff --git a/litellm-rust/crates/llms/src/openai/responses/mod.rs b/litellm-rust/crates/llms/src/openai/responses/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/openai/responses/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/core/src/llms/openai/responses/transformation.rs b/litellm-rust/crates/llms/src/openai/responses/transformation.rs similarity index 91% rename from litellm-rust/crates/core/src/llms/openai/responses/transformation.rs rename to litellm-rust/crates/llms/src/openai/responses/transformation.rs index ddd160af7a4..f01ec4ad146 100644 --- a/litellm-rust/crates/core/src/llms/openai/responses/transformation.rs +++ b/litellm-rust/crates/llms/src/openai/responses/transformation.rs @@ -1,8 +1,8 @@ use litellm_types::responses::streaming_websocket::{ResponsesWsEvent, ResponsesWsTransformResult}; -use crate::responses::{ - Error, - websocket::{ResponsesWebSocketProviderConfig, enforce_model}, +use crate::base_llm::{ + chat::transformation::Error, + responses::transformation::{ResponsesWebSocketProviderConfig, enforce_model}, }; pub struct OpenAiResponsesApiConfig; diff --git a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/mod.rs b/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/mod.rs deleted file mode 100644 index ba63992f3cb..00000000000 --- a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod messages;