diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index ea98a5f6b06..3b9ff62fbaa 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2025,19 +2025,15 @@ dependencies = [ name = "litellm-core" version = "0.1.0" dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-types", "base64 0.22.1", "bytes", - "data-url", "futures-util", "litellm-auth", "litellm-auth-aws", - "litellm-auth-azure", - "litellm-auth-gcp", "litellm-callbacks", - "litellm-framing", - "litellm-providers", + "litellm-core-utils", + "litellm-llms", + "litellm-types", "mime_guess", "moka", "rand 0.8.7", @@ -2048,8 +2044,6 @@ dependencies = [ "rustls-native-certs", "serde", "serde_json", - "serde_path_to_error", - "serde_with", "sha2 0.10.9", "strum", "subtle", @@ -2061,6 +2055,19 @@ dependencies = [ "veil", ] +[[package]] +name = "litellm-core-utils" +version = "0.1.0" +dependencies = [ + "litellm-types", + "serde", + "serde_json", + "serde_path_to_error", + "serde_with", + "thiserror 2.0.19", + "url", +] + [[package]] name = "litellm-framing" version = "0.1.0" @@ -2091,15 +2098,33 @@ dependencies = [ ] [[package]] -name = "litellm-providers" +name = "litellm-llms" version = "0.1.0" dependencies = [ + "aws-smithy-eventstream", + "aws-smithy-types", + "base64 0.22.1", + "bytes", + "data-url", + "futures-util", "litellm-auth", "litellm-auth-aws", + "litellm-auth-azure", + "litellm-auth-gcp", + "litellm-callbacks", + "litellm-core-utils", + "litellm-framing", + "litellm-types", + "reqwest 0.12.28", "rstest", "serde", "serde_json", + "serde_path_to_error", + "serde_with", "thiserror 2.0.19", + "time", + "tokio", + "url", ] [[package]] @@ -2113,7 +2138,9 @@ dependencies = [ "litellm-callbacks-legacy", "litellm-core", "litellm-host-python", + "litellm-llms", "litellm-token-counter", + "litellm-types", "pyo3", "pyo3-async-runtimes", "rstest", @@ -2140,6 +2167,14 @@ dependencies = [ "unicode-normalization-alignments", ] +[[package]] +name = "litellm-types" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "litemap" version = "0.8.2" diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 851ef91a1fb..f8377138050 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -17,7 +17,9 @@ litellm-auth = { path = "crates/auth" } litellm-auth-aws = { path = "crates/auth-aws" } litellm-auth-azure = { path = "crates/auth-azure" } litellm-auth-gcp = { path = "crates/auth-gcp" } -litellm-providers = { path = "crates/providers" } +litellm-llms = { path = "crates/llms" } +litellm-types = { path = "crates/types" } +litellm-core-utils = { path = "crates/core-utils" } litellm-cache = { path = "crates/cache" } litellm-cache-memory = { path = "crates/cache-memory" } litellm-token-counter = { path = "crates/token-counter" } diff --git a/litellm-rust/crates/providers/Cargo.toml b/litellm-rust/crates/core-utils/Cargo.toml similarity index 59% rename from litellm-rust/crates/providers/Cargo.toml rename to litellm-rust/crates/core-utils/Cargo.toml index e1c8f2c50d4..109c3312727 100644 --- a/litellm-rust/crates/providers/Cargo.toml +++ b/litellm-rust/crates/core-utils/Cargo.toml @@ -1,16 +1,15 @@ [package] -name = "litellm-providers" +name = "litellm-core-utils" version = "0.1.0" edition.workspace = true license.workspace = true repository.workspace = true [dependencies] -litellm-auth.workspace = true -litellm-auth-aws.workspace = true +litellm-types.workspace = true serde.workspace = true serde_json.workspace = true +serde_path_to_error = "0.1" +serde_with.workspace = true thiserror.workspace = true - -[dev-dependencies] -rstest.workspace = true +url.workspace = true diff --git a/litellm-rust/crates/core/src/call_arguments.rs b/litellm-rust/crates/core-utils/src/call_arguments.rs similarity index 98% rename from litellm-rust/crates/core/src/call_arguments.rs rename to litellm-rust/crates/core-utils/src/call_arguments.rs index eb1dcd8deb7..31fe1978f2c 100644 --- a/litellm-rust/crates/core/src/call_arguments.rs +++ b/litellm-rust/crates/core-utils/src/call_arguments.rs @@ -8,7 +8,7 @@ use serde_json::{Map, Value}; pub struct CallArguments(Map); impl CallArguments { - pub(crate) fn select(&self, names: &[&str]) -> Map { + pub fn select(&self, names: &[&str]) -> Map { self.iter() .filter(|(name, _)| names.contains(&name.as_str())) .map(|(name, value)| (name.clone(), value.clone())) diff --git a/litellm-rust/crates/providers/src/chat/response_utils.rs b/litellm-rust/crates/core-utils/src/core_helpers.rs similarity index 88% rename from litellm-rust/crates/providers/src/chat/response_utils.rs rename to litellm-rust/crates/core-utils/src/core_helpers.rs index 1ada5d43980..cc9fc7a6687 100644 --- a/litellm-rust/crates/providers/src/chat/response_utils.rs +++ b/litellm-rust/crates/core-utils/src/core_helpers.rs @@ -2,7 +2,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; -use super::types::{ChatCompletionsUsage, PromptTokensDetails}; +use litellm_types::utils::{ChatCompletionsUsage, PromptTokensDetails}; /// OpenAI finish reasons, mirroring Python's `_FINISH_REASON_MAP` for the /// reasons the providers on this route can emit. Python warns and falls back to @@ -54,6 +54,17 @@ pub fn unix_now() -> u64 { .map_or(0, |elapsed| elapsed.as_secs()) } +pub fn json_type_name(value: &serde_json::Value) -> &'static str { + match value { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "boolean", + serde_json::Value::Number(_) => "number", + serde_json::Value::String(_) => "string", + serde_json::Value::Array(_) => "array", + serde_json::Value::Object(_) => "object", + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/litellm-rust/crates/core/src/litellm_core_utils/get_llm_provider_logic.rs b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs similarity index 59% rename from litellm-rust/crates/core/src/litellm_core_utils/get_llm_provider_logic.rs rename to litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs index 5958e8ac613..6333eedebfc 100644 --- a/litellm-rust/crates/core/src/litellm_core_utils/get_llm_provider_logic.rs +++ b/litellm-rust/crates/core-utils/src/get_llm_provider_logic.rs @@ -1,4 +1,36 @@ -pub use litellm_providers::provider_resolution::{CustomLlmProvider, get_custom_llm_provider}; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CustomLlmProvider<'a> { + pub model: &'a str, + pub custom_llm_provider: &'a str, +} + +pub fn get_custom_llm_provider<'a>( + model: &'a str, + custom_llm_provider: Option<&'a str>, +) -> Option> { + if let Some(custom_llm_provider) = custom_llm_provider.filter(|provider| !provider.is_empty()) { + return Some(CustomLlmProvider { + model: strip_custom_llm_provider_prefix(model, custom_llm_provider), + custom_llm_provider, + }); + } + + let (custom_llm_provider, model) = model.split_once('/')?; + if custom_llm_provider.is_empty() || model.is_empty() { + return None; + } + Some(CustomLlmProvider { + model, + custom_llm_provider, + }) +} + +fn strip_custom_llm_provider_prefix<'a>(model: &'a str, custom_llm_provider: &str) -> &'a str { + model + .strip_prefix(custom_llm_provider) + .and_then(|model| model.strip_prefix('/')) + .unwrap_or(model) +} #[cfg(test)] mod tests { diff --git a/litellm-rust/crates/core-utils/src/lib.rs b/litellm-rust/crates/core-utils/src/lib.rs new file mode 100644 index 00000000000..a8895cccf2c --- /dev/null +++ b/litellm-rust/crates/core-utils/src/lib.rs @@ -0,0 +1,7 @@ +pub mod call_arguments; +pub mod core_helpers; +pub mod get_llm_provider_logic; +pub mod params; +pub mod prompt_templates; +pub mod serde_compat; +pub mod url_utils; diff --git a/litellm-rust/crates/core/src/params.rs b/litellm-rust/crates/core-utils/src/params.rs similarity index 100% rename from litellm-rust/crates/core/src/params.rs rename to litellm-rust/crates/core-utils/src/params.rs diff --git a/litellm-rust/crates/providers/src/chat/conversation.rs b/litellm-rust/crates/core-utils/src/prompt_templates/factory.rs similarity index 94% rename from litellm-rust/crates/providers/src/chat/conversation.rs rename to litellm-rust/crates/core-utils/src/prompt_templates/factory.rs index 587b7ea2a16..2c4921d26be 100644 --- a/litellm-rust/crates/providers/src/chat/conversation.rs +++ b/litellm-rust/crates/core-utils/src/prompt_templates/factory.rs @@ -10,8 +10,10 @@ //! `_bedrock_converse_messages_pt` for the text-only surface this route //! accepts; anything richer is declined upstream by the capability gate. -use super::types::{ChatMessage, ChatMessageContent}; -use crate::chat::EMPTY_TEXT_PLACEHOLDER; +use litellm_types::llms::openai::{ChatMessage, ChatMessageContent}; + +pub const EMPTY_TEXT_PLACEHOLDER: &str = + "[System: Empty message content sanitised to satisfy protocol]"; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum TurnRole { @@ -203,8 +205,10 @@ mod tests { {"role": "assistant", "content": " "}, {"role": "user", "content": "real"} ]))); - assert_eq!(conversation.turns[0].texts, vec![EMPTY_TEXT_PLACEHOLDER]); - assert_eq!(conversation.turns[1].texts, vec![EMPTY_TEXT_PLACEHOLDER]); + // Must equal `_EMPTY_TEXT_PLACEHOLDER` in litellm/litellm_core_utils/prompt_templates/factory.py + let placeholder = "[System: Empty message content sanitised to satisfy protocol]"; + assert_eq!(conversation.turns[0].texts, vec![placeholder]); + assert_eq!(conversation.turns[1].texts, vec![placeholder]); } #[test] diff --git a/litellm-rust/crates/core-utils/src/prompt_templates/mod.rs b/litellm-rust/crates/core-utils/src/prompt_templates/mod.rs new file mode 100644 index 00000000000..a106d20eaff --- /dev/null +++ b/litellm-rust/crates/core-utils/src/prompt_templates/mod.rs @@ -0,0 +1 @@ +pub mod factory; diff --git a/litellm-rust/crates/core/src/serde_compat.rs b/litellm-rust/crates/core-utils/src/serde_compat.rs similarity index 98% rename from litellm-rust/crates/core/src/serde_compat.rs rename to litellm-rust/crates/core-utils/src/serde_compat.rs index 3ec869b40e2..bb2648eb0be 100644 --- a/litellm-rust/crates/core/src/serde_compat.rs +++ b/litellm-rust/crates/core-utils/src/serde_compat.rs @@ -2,8 +2,8 @@ use serde::{Deserialize, Deserializer, de::Error}; use serde_json::Value; use serde_with::DeserializeAs; -pub(crate) struct LaxI64; -pub(crate) struct FiniteF64; +pub struct LaxI64; +pub struct FiniteF64; impl<'de> DeserializeAs<'de, i64> for LaxI64 { fn deserialize_as>(deserializer: D) -> Result { diff --git a/litellm-rust/crates/core/src/url_utils.rs b/litellm-rust/crates/core-utils/src/url_utils.rs similarity index 88% rename from litellm-rust/crates/core/src/url_utils.rs rename to litellm-rust/crates/core-utils/src/url_utils.rs index b8d82b7a04a..1f690752c7a 100644 --- a/litellm-rust/crates/core/src/url_utils.rs +++ b/litellm-rust/crates/core-utils/src/url_utils.rs @@ -3,33 +3,30 @@ use std::marker::PhantomData; use url::Url; #[derive(Debug, thiserror::Error)] -pub(crate) enum ApiUrlError { +pub enum ApiUrlError { #[error("invalid URL: {0}")] Parse(#[from] url::ParseError), #[error("URL cannot be used as a base")] CannotBeBase, } -pub(crate) struct Base; -pub(crate) struct Complete; +pub struct Base; +pub struct Complete; -pub(crate) struct ApiUrl { +pub struct ApiUrl { url: Url, state: PhantomData, } impl ApiUrl { - pub(crate) fn parse(value: &str) -> Result { + pub fn parse(value: &str) -> Result { Ok(Self { url: Url::parse(value.trim())?, state: PhantomData, }) } - pub(crate) fn complete_path( - mut self, - target: &[&str], - ) -> Result, ApiUrlError> { + pub fn complete_path(mut self, target: &[&str]) -> Result, ApiUrlError> { let existing: Vec = self .url .path_segments() @@ -59,7 +56,7 @@ impl ApiUrl { } impl ApiUrl { - pub(crate) fn append_query_pairs<'a>( + pub fn append_query_pairs<'a>( mut self, pairs: impl IntoIterator, ) -> Self { @@ -67,7 +64,7 @@ impl ApiUrl { self } - pub(crate) fn into_string(self) -> String { + pub fn into_string(self) -> String { self.url.into() } } diff --git a/litellm-rust/crates/core/AGENTS.md b/litellm-rust/crates/core/AGENTS.md index 7a7e988b07c..449c3e647f7 100644 --- a/litellm-rust/crates/core/AGENTS.md +++ b/litellm-rust/crates/core/AGENTS.md @@ -1,27 +1,14 @@ litellm-core is the LiteLLM SDK in Rust — it makes the LLM call. Each top-level call is a module under `src//` exposing a public entrypoint named after the route (`messages::messages()`, the Rust equivalent of `litellm.messages()`): you call it and get a typed non-streaming response back. -A route module owns the call entrypoint, runtime types, provider/auth/URL resolution, and the handler that performs the HTTP call. Provider code and base config traits live under `src/llms/`, mirroring their Python source paths. This applies to every API surface: shared orchestration stays in its route module (`ocr/`, `chat_completions/`, `messages/`, `audio_transcription/`, or `responses/`), while provider transformations live under the corresponding Python-mirrored `llms//` path. Import implementations directly from their canonical paths; do not add a `src/providers/` layer or compatibility re-exports. Shared provider resolution lives under `src/litellm_core_utils/get_llm_provider_logic.rs`. Handlers belong in core, never in a host crate +## Crate layering -Not here: serving HTTP (axum routes, extractors), config file reading, rollout state, databases, or callback execution of any kind. Core runs each route as a machine that yields host operations and call events; which integrations consume those events is the host's business. Env reads are limited to credential fallback in a route's `prepare.rs`. +Each crate mirrors one top-level Python package, so a Rust path reads as its Python path with the crate name in place of the package directory. Dependencies only point down: -Routes (messages, ocr, realtime) and providers (anthropic, mistral, openai) are modules, not crates. +- `litellm-types` mirrors `litellm/types/`: pure serde data, no I/O +- `litellm-core-utils` mirrors `litellm/litellm_core_utils/`: pure helpers (provider resolution, prompt factory, call arguments), no network I/O +- `litellm-llms` mirrors `litellm/llms/`: `base_llm//transformation.rs`, `//transformation.rs`, and `custom_httpx/` (HTTP helpers and the OCR request handler) +- `litellm-core` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...): entrypoints, route request types, provider dispatch, the route machine, and hooks -## Python/Rust transformation pairs +A route module owns the call entrypoint, route request types (`*Request<'a>`), credential fallback, provider dispatch, and the handler glue that runs a provider config. Provider code never imports from core; when it needs the caller's hooks mid-call it goes through `litellm_llms::custom_httpx::llm_http_handler::CallHooks`, which each route implements over its host. Import every item from its canonical path. Never re-export another crate's items or give an item a second public path; the only re-export allowed is a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`). Handlers belong in core or llms, never in a host crate -Use the base OCR and Mistral OCR pairs as the reference when aligning transformations. Derive `src/.rs` from `litellm/.py`, preserving meaningful basenames such as `messages_transformation` - -Keep corresponding operation names and parameter names when their responsibilities match. Rust types retain the Python semantic name with Rust acronym casing (`BaseOCRConfig` / `BaseOcrConfig`, `MistralOCRConfig` / `MistralOcrConfig`). Private Python helpers can drop their leading underscore. Give Rust adapter helpers distinct responsibility names rather than duplicating trait method names - -Order OCR config methods as supported parameters, credential metadata and connection resolution, health-check input, parameter mapping, environment validation, URL construction, request transformation, async request transformation, response transformation, async response transformation, and error conversion. Put constants and data types before the config, private helpers after it in operation order, and tests last. Rust-only trait hooks follow the corresponding Python methods - -Use trait defaults for unchanged inherited behavior and explicit delegation for shared provider behavior. Keep typed inputs, ownership, `Result`, and async I/O idiomatic. A matching path or symbol identifies the counterpart, not a claim of full behavioral parity - -Use named `#[rstest]` cases for independent input/output scenarios instead of loops or repeated calls in one test. Inject reusable setup with `#[fixture]` arguments and use `#[with(...)]` for fixture overrides. Keep assertions about the same result together - -For base OCR, Python response models correspond to `src/ocr/types.rs`; Rust context/environment types support the runtime. `BaseOcrConfig::prepare_request` corresponds to Python's HTTP-handler preparation rather than a `BaseOCRConfig` method, and `validate_request_body` is a Rust-only hook - -For Mistral, `async_transform_ocr_request` uses the base default in both languages. `resolve_headers` and `build_ocr_url` implement the respective environment and URL operations, and `normalize_response` implements the typed part of response transformation. Existing auth key/header handling and top-level response-extra preservation differ between languages; layout refactors must preserve those behaviors and verify them with the existing tests - -For non-OCR pairs, order corresponding methods as parameter support/mapping, environment validation, URL construction, request transformation, and response transformation, followed by Rust-only runtime hooks. Auth resolution remains split between configs and route preparation. Chat `supported_openai_param_mappings` describes accepted OpenAI/provider name pairs, unlike Python's `get_supported_openai_params` name list. Audio `map_transcription_params` remains a Rust filtering helper - -Azure Messages maps to `llms/azure_ai/anthropic/messages_transformation.py`; Bedrock Converse maps to `llms/bedrock/chat/converse_transformation.py`. `AnthropicConfig`, `AmazonConverseConfig`, and the non-OCR base traits are partial ports. `OpenAiResponsesApiConfig` currently implements only the WebSocket surface. Preserve their acceptance gates, passthrough behavior, and host fallback contracts when aligning layout +Not here: serving HTTP (axum routes, extractors), config file reading, rollout state, databases, or callback execution of any kind. Core runs each route as a machine that yields host operations and call events; which integrations consume those events is the host's business. diff --git a/litellm-rust/crates/core/Cargo.toml b/litellm-rust/crates/core/Cargo.toml index b9382ac7afd..db6cfc4b340 100644 --- a/litellm-rust/crates/core/Cargo.toml +++ b/litellm-rust/crates/core/Cargo.toml @@ -7,17 +7,15 @@ repository.workspace = true autotests = false [dependencies] +litellm-types.workspace = true +litellm-core-utils.workspace = true litellm-callbacks.workspace = true bytes.workspace = true futures-util.workspace = true base64.workspace = true -data-url = "0.3.2" litellm-auth.workspace = true litellm-auth-aws.workspace = true -litellm-auth-azure.workspace = true -litellm-auth-gcp.workspace = true -litellm-providers.workspace = true -litellm-framing.workspace = true +litellm-llms.workspace = true moka.workspace = true mime_guess = "2.0.5" rand.workspace = true @@ -26,8 +24,6 @@ rustls.workspace = true rustls-native-certs.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } -serde_with.workspace = true -serde_path_to_error = "0.1" strum.workspace = true subtle.workspace = true tokio = { workspace = true, features = ["sync"] } @@ -39,7 +35,6 @@ url.workspace = true veil.workspace = true [dev-dependencies] -aws-smithy-eventstream = "=0.61.1" -aws-smithy-types = "1.6.1" +litellm-llms = { workspace = true, features = ["test-support"] } rstest.workspace = true rstest_reuse.workspace = true diff --git a/litellm-rust/crates/core/src/audio_transcription/error.rs b/litellm-rust/crates/core/src/audio_transcription/error.rs index ab194173b67..39b08e882f5 100644 --- a/litellm-rust/crates/core/src/audio_transcription/error.rs +++ b/litellm-rust/crates/core/src/audio_transcription/error.rs @@ -1,3 +1,5 @@ +use litellm_llms::base_llm::chat::transformation::Error as LlmError; + #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { #[error("expected {expected}, got {actual}")] @@ -18,29 +20,22 @@ pub enum Error { #[error(transparent)] Auth(#[from] litellm_auth::Error), #[error(transparent)] - Transport(#[from] crate::transport::Error), + Transport(#[from] litellm_llms::custom_httpx::transport::Error), #[error(transparent)] - Headers(#[from] crate::http_utils::HeaderError), + Headers(#[from] litellm_llms::custom_httpx::http_handler::HeaderError), #[error(transparent)] Aws(#[from] litellm_auth_aws::Error), } -impl From for Error { - fn from(error: litellm_providers::audio_transcription::Error) -> Self { +impl From for Error { + fn from(error: LlmError) -> Self { match error { - litellm_providers::audio_transcription::Error::InvalidType { expected, actual } => { - Self::InvalidType { expected, actual } - } - litellm_providers::audio_transcription::Error::MissingField(field) => { - Self::MissingField(field) - } - litellm_providers::audio_transcription::Error::InvalidRequest(message) => { - Self::InvalidRequest(message) - } - litellm_providers::audio_transcription::Error::InvalidResponse(message) => { - Self::InvalidResponse(message) - } - litellm_providers::audio_transcription::Error::Auth(error) => Self::Auth(error), + LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual }, + LlmError::MissingField(field) => Self::MissingField(field), + LlmError::InvalidRequest(message) => Self::InvalidRequest(message), + LlmError::InvalidResponse(message) => Self::InvalidResponse(message), + LlmError::Unsupported(reason) => Self::Unsupported(reason), + LlmError::Auth(error) => Self::Auth(error), } } } diff --git a/litellm-rust/crates/core/src/audio_transcription/handler.rs b/litellm-rust/crates/core/src/audio_transcription/handler.rs index a7ab93ccd48..0704f9391b0 100644 --- a/litellm-rust/crates/core/src/audio_transcription/handler.rs +++ b/litellm-rust/crates/core/src/audio_transcription/handler.rs @@ -1,7 +1,8 @@ +use litellm_llms::custom_httpx::http_handler::{http_request, truncate_error_body}; use serde_json::Value; -use super::{Error, client::http_client, types::ProviderAudioTranscriptionRequest}; -use crate::http_utils::{http_request, truncate_error_body}; +use super::{Error, client::http_client}; +use crate::audio_transcription::types::ProviderAudioTranscriptionRequest; pub async fn execute_audio_transcription_provider_call( request: ProviderAudioTranscriptionRequest, @@ -16,19 +17,24 @@ pub async fn execute_audio_transcription_provider_call( if let Some(duration) = request.timeout { request_builder = request_builder.timeout(duration); } - let response = http_request(request_builder) - .await - .map_err(|error| Error::Transport(crate::transport::Error::Network(error.to_string())))?; + let response = http_request(request_builder).await.map_err(|error| { + Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( + error.to_string(), + )) + })?; let status = response.status(); - let text = response - .text() - .await - .map_err(|error| Error::Transport(crate::transport::Error::Network(error.to_string())))?; + let text = response.text().await.map_err(|error| { + Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( + error.to_string(), + )) + })?; if !status.is_success() { - return Err(Error::Transport(crate::transport::Error::Http { - status: status.as_u16(), - body: truncate_error_body(&text), - })); + return Err(Error::Transport( + litellm_llms::custom_httpx::transport::Error::Http { + status: status.as_u16(), + body: truncate_error_body(&text), + }, + )); } let response_json = serde_json::from_str(&text) .map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?; @@ -45,7 +51,7 @@ async fn signed_headers( use std::{collections::BTreeMap, time::SystemTime}; use litellm_auth_aws::{aws_auth_config, resolve_credentials, sign_bedrock_post}; - use litellm_providers::base_llm::audio_transcription::transformation::AudioTranscriptionAuth; + use litellm_llms::base_llm::audio_transcription::transformation::AudioTranscriptionAuth; let AudioTranscriptionAuth::AwsSigV4 { region, .. } = &request.auth else { return Ok(request.upstream_headers.clone()); diff --git a/litellm-rust/crates/core/src/audio_transcription/mod.rs b/litellm-rust/crates/core/src/audio_transcription/mod.rs index 5037fa2322e..af9c398c065 100644 --- a/litellm-rust/crates/core/src/audio_transcription/mod.rs +++ b/litellm-rust/crates/core/src/audio_transcription/mod.rs @@ -1,13 +1,14 @@ mod error; +pub mod types; pub use error::Error; mod client; mod handler; mod prepare; pub use handler::execute_audio_transcription_provider_call; -pub use litellm_providers::audio_transcription::types; pub use prepare::prepare_audio_transcription_provider_call; use serde_json::Value; -pub use types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}; + +use crate::audio_transcription::types::AudioTranscriptionRequest; pub async fn audio_transcription(request: AudioTranscriptionRequest<'_>) -> Result { execute_audio_transcription_provider_call(prepare_audio_transcription_provider_call(request)?) diff --git a/litellm-rust/crates/core/src/audio_transcription/prepare.rs b/litellm-rust/crates/core/src/audio_transcription/prepare.rs index beecdab9615..193122db733 100644 --- a/litellm-rust/crates/core/src/audio_transcription/prepare.rs +++ b/litellm-rust/crates/core/src/audio_transcription/prepare.rs @@ -1,17 +1,15 @@ -use litellm_providers::{ +use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; +use litellm_llms::{ base_llm::audio_transcription::transformation::{ AudioTranscriptionAuth, BaseAudioTranscriptionConfig, }, bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG, + custom_httpx::http_handler::{has_header, string_headers}, }; -use super::{ - Error, - types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest}, -}; -use crate::{ - http_utils::{has_header, string_headers}, - litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}, +use super::Error; +use crate::audio_transcription::types::{ + AudioTranscriptionRequest, ProviderAudioTranscriptionRequest, }; fn provider_config(provider: &str) -> Option<&'static dyn BaseAudioTranscriptionConfig> { diff --git a/litellm-rust/crates/core/src/audio_transcription/tests.rs b/litellm-rust/crates/core/src/audio_transcription/tests.rs index d6491ca8ce0..8ccf7a07a0f 100644 --- a/litellm-rust/crates/core/src/audio_transcription/tests.rs +++ b/litellm-rust/crates/core/src/audio_transcription/tests.rs @@ -6,7 +6,8 @@ use std::{ use serde_json::{Map, json}; -use super::{audio_transcription, types::AudioTranscriptionRequest}; +use super::audio_transcription; +use crate::audio_transcription::types::AudioTranscriptionRequest; #[tokio::test] async fn bedrock_request_is_signed_and_contains_audio() { diff --git a/litellm-rust/crates/providers/src/audio_transcription/types.rs b/litellm-rust/crates/core/src/audio_transcription/types.rs similarity index 71% rename from litellm-rust/crates/providers/src/audio_transcription/types.rs rename to litellm-rust/crates/core/src/audio_transcription/types.rs index d17d5067de5..ca09dd945be 100644 --- a/litellm-rust/crates/providers/src/audio_transcription/types.rs +++ b/litellm-rust/crates/core/src/audio_transcription/types.rs @@ -1,11 +1,9 @@ use std::time::Duration; -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; - -use crate::base_llm::audio_transcription::transformation::{ +use litellm_llms::base_llm::audio_transcription::transformation::{ AudioTranscriptionAuth, BaseAudioTranscriptionConfig, }; +use serde_json::{Map, Value}; pub struct AudioTranscriptionRequest<'a> { pub model: &'a str, @@ -52,21 +50,3 @@ impl ProviderAudioTranscriptionRequest { Self { body, ..self } } } - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct AudioTranscriptionRequestData { - pub body: Value, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct AudioTranscriptionResponseData { - pub text: String, -} - -impl AudioTranscriptionResponseData { - pub fn into_json(self) -> Value { - serde_json::json!({ - "text": self.text, - }) - } -} diff --git a/litellm-rust/crates/core/src/chat_completions/common_utils.rs b/litellm-rust/crates/core/src/chat_completions/common_utils.rs index 309cc781cc0..cc9459793df 100644 --- a/litellm-rust/crates/core/src/chat_completions/common_utils.rs +++ b/litellm-rust/crates/core/src/chat_completions/common_utils.rs @@ -1,20 +1,19 @@ -use litellm_providers::{ +use litellm_llms::{ anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG, base_llm::chat::transformation::BaseConfig, + bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, + custom_httpx::http_handler::string_headers as shared_string_headers, }; use serde_json::{Map, Value}; use super::Error; -use crate::http_utils::string_headers as shared_string_headers; const HEADER_CONTEXT: &str = "chat completions"; pub(super) fn chat_completions_provider_config(provider: &str) -> Option<&'static dyn BaseConfig> { match provider { "anthropic" => Some(&ANTHROPIC_CHAT_COMPLETIONS_CONFIG), - "bedrock" => Some( - &litellm_providers::bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, - ), + "bedrock" => Some(&BEDROCK_CHAT_COMPLETIONS_CONFIG), _ => None, } } diff --git a/litellm-rust/crates/core/src/chat_completions/error.rs b/litellm-rust/crates/core/src/chat_completions/error.rs index 95da97125d7..39b08e882f5 100644 --- a/litellm-rust/crates/core/src/chat_completions/error.rs +++ b/litellm-rust/crates/core/src/chat_completions/error.rs @@ -1,3 +1,5 @@ +use litellm_llms::base_llm::chat::transformation::Error as LlmError; + #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { #[error("expected {expected}, got {actual}")] @@ -18,25 +20,22 @@ pub enum Error { #[error(transparent)] Auth(#[from] litellm_auth::Error), #[error(transparent)] - Transport(#[from] crate::transport::Error), + Transport(#[from] litellm_llms::custom_httpx::transport::Error), #[error(transparent)] - Headers(#[from] crate::http_utils::HeaderError), + Headers(#[from] litellm_llms::custom_httpx::http_handler::HeaderError), #[error(transparent)] Aws(#[from] litellm_auth_aws::Error), } -impl From for Error { - fn from(error: litellm_providers::chat::Error) -> Self { +impl From for Error { + fn from(error: LlmError) -> Self { match error { - litellm_providers::chat::Error::MissingField(field) => Self::MissingField(field), - litellm_providers::chat::Error::InvalidRequest(message) => { - Self::InvalidRequest(message) - } - litellm_providers::chat::Error::InvalidResponse(message) => { - Self::InvalidResponse(message) - } - litellm_providers::chat::Error::Unsupported(reason) => Self::Unsupported(reason), - litellm_providers::chat::Error::Auth(error) => Self::Auth(error), + LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual }, + LlmError::MissingField(field) => Self::MissingField(field), + LlmError::InvalidRequest(message) => Self::InvalidRequest(message), + LlmError::InvalidResponse(message) => Self::InvalidResponse(message), + LlmError::Unsupported(reason) => Self::Unsupported(reason), + LlmError::Auth(error) => Self::Auth(error), } } } diff --git a/litellm-rust/crates/core/src/chat_completions/handler.rs b/litellm-rust/crates/core/src/chat_completions/handler.rs index d9939177f31..034408bdf17 100644 --- a/litellm-rust/crates/core/src/chat_completions/handler.rs +++ b/litellm-rust/crates/core/src/chat_completions/handler.rs @@ -1,16 +1,14 @@ -use litellm_providers::base_llm::chat::transformation::ChatCompletionsAuth; +use litellm_llms::{ + base_llm::chat::transformation::{ChatCompletionsAuth, ProviderChatResponseData}, + custom_httpx::http_handler::{http_request, truncate_error_body}, +}; +use litellm_types::utils::ChatCompletionsResponse; use serde_json::Value; -use super::{ - Error, - client::http_client, - prepare::prepare_provider_request, - types::{ - ChatCompletionsResponse, ProviderChatCompletionsRequest, ProviderChatResponseData, - ResolvedChatCompletionsRequest, - }, +use super::{Error, client::http_client, prepare::prepare_provider_request}; +use crate::chat_completions::types::{ + ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest, }; -use crate::http_utils::{http_request, truncate_error_body}; pub(super) async fn execute_chat_completions_provider_call( request: ResolvedChatCompletionsRequest<'_>, @@ -36,23 +34,30 @@ pub(super) async fn execute_chat_completions_provider_call( // so the host can still serve it. Everything else here, a timeout // above all, may have reached the provider and been answered. if err.is_connect() || err.is_builder() { - Error::Transport(crate::transport::Error::Connect(err.to_string())) + Error::Transport(litellm_llms::custom_httpx::transport::Error::Connect( + err.to_string(), + )) } else { - Error::Transport(crate::transport::Error::Network(err.to_string())) + Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( + err.to_string(), + )) } })?; let status = response.status(); - let text = response - .text() - .await - .map_err(|err| Error::Transport(crate::transport::Error::Network(err.to_string())))?; + let text = response.text().await.map_err(|err| { + Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( + err.to_string(), + )) + })?; if !status.is_success() { - return Err(Error::Transport(crate::transport::Error::Http { - status: status.as_u16(), - body: truncate_error_body(&text), - })); + return Err(Error::Transport( + litellm_llms::custom_httpx::transport::Error::Http { + status: status.as_u16(), + body: truncate_error_body(&text), + }, + )); } let body: Value = serde_json::from_str(&text).map_err(|err| { @@ -77,7 +82,9 @@ pub(super) async fn execute_chat_completions_provider_call( pub(super) fn as_response_error(err: Error) -> Error { match err { already @ (Error::InvalidResponse(_) - | Error::Transport(crate::transport::Error::Http { .. })) => already, + | Error::Transport(litellm_llms::custom_httpx::transport::Error::Http { + .. + })) => already, other => Error::InvalidResponse(other.to_string()), } } diff --git a/litellm-rust/crates/core/src/chat_completions/mod.rs b/litellm-rust/crates/core/src/chat_completions/mod.rs index 2fd619f9f93..81d35044d08 100644 --- a/litellm-rust/crates/core/src/chat_completions/mod.rs +++ b/litellm-rust/crates/core/src/chat_completions/mod.rs @@ -7,18 +7,18 @@ //! calls the provider, and returns a typed OpenAI-shaped response. mod error; +pub mod types; pub use error::Error; mod client; mod common_utils; -pub use litellm_providers::chat::{conversation, response_utils}; pub(crate) mod handler; mod prepare; -pub mod streaming; use handler::execute_chat_completions_provider_call; -pub use litellm_providers::chat::types; +use litellm_types::utils::ChatCompletionsResponse; use prepare::{parse_messages, resolve_provider_config, resolve_request}; use serde_json::{Map, Value}; -use types::{ChatCompletionsRequest, ChatCompletionsResponse}; + +use crate::chat_completions::types::ChatCompletionsRequest; pub async fn chat_completions( request: ChatCompletionsRequest<'_>, diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/core/src/chat_completions/prepare.rs index d7b2a58596f..d408ea6574e 100644 --- a/litellm-rust/crates/core/src/chat_completions/prepare.rs +++ b/litellm-rust/crates/core/src/chat_completions/prepare.rs @@ -1,17 +1,17 @@ -use litellm_providers::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; +use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; +use litellm_llms::{ + base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}, + custom_httpx::http_handler::has_header, +}; +use litellm_types::llms::openai::ChatMessage; use serde_json::Value; use super::{ Error, common_utils::{chat_completions_provider_config, string_headers}, - types::{ - ChatCompletionsRequest, ChatMessage, ProviderChatCompletionsRequest, - ResolvedChatCompletionsRequest, - }, }; -use crate::{ - http_utils::has_header, - litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}, +use crate::chat_completions::types::{ + ChatCompletionsRequest, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest, }; pub(super) fn resolve_provider_config<'a>( diff --git a/litellm-rust/crates/core/src/chat_completions/tests.rs b/litellm-rust/crates/core/src/chat_completions/tests.rs index e9f1451022e..cbc4995ce0d 100644 --- a/litellm-rust/crates/core/src/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/chat_completions/tests.rs @@ -1,11 +1,11 @@ -use litellm_providers::base_llm::chat::transformation::ChatCompletionsAuth; +use litellm_llms::base_llm::chat::transformation::ChatCompletionsAuth; use serde_json::{Map, Value, json}; use super::{ Error, prepare::{prepare_provider_request, resolve_request}, - types::{ChatCompletionsRequest, ProviderChatCompletionsRequest}, }; +use crate::chat_completions::types::{ChatCompletionsRequest, ProviderChatCompletionsRequest}; fn prepare_chat_completions_call( request: ChatCompletionsRequest<'_>, @@ -265,7 +265,7 @@ fn rejects_non_string_extra_headers() { call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))])); assert_eq!( decline(call), - Error::Headers(crate::http_utils::HeaderError { + Error::Headers(litellm_llms::custom_httpx::http_handler::HeaderError { context: "chat completions", name: "x-trace".to_string(), actual: "number", @@ -771,7 +771,10 @@ mod round_trip { assert!( matches!( err, - Error::Transport(crate::transport::Error::Http { status: 429, .. }) + Error::Transport(litellm_llms::custom_httpx::transport::Error::Http { + status: 429, + .. + }) ), "expected a 429, got {err:?}" ); @@ -796,7 +799,10 @@ mod round_trip { .await .expect_err("nothing is listening"); assert!( - matches!(err, Error::Transport(crate::transport::Error::Connect(_))), + matches!( + err, + Error::Transport(litellm_llms::custom_httpx::transport::Error::Connect(_)) + ), "expected a pre-send connect failure, got {err:?}" ); } @@ -819,11 +825,16 @@ mod round_trip { } // An upstream status is already unambiguous, so it survives intact. assert!(matches!( - as_response_error(Error::Transport(crate::transport::Error::Http { + as_response_error(Error::Transport( + litellm_llms::custom_httpx::transport::Error::Http { + status: 500, + body: "boom".to_string() + } + )), + Error::Transport(litellm_llms::custom_httpx::transport::Error::Http { status: 500, - body: "boom".to_string() - })), - Error::Transport(crate::transport::Error::Http { status: 500, .. }) + .. + }) )); } } diff --git a/litellm-rust/crates/core/src/chat_completions/types.rs b/litellm-rust/crates/core/src/chat_completions/types.rs new file mode 100644 index 00000000000..882611d5862 --- /dev/null +++ b/litellm-rust/crates/core/src/chat_completions/types.rs @@ -0,0 +1,44 @@ +use std::time::Duration; + +use litellm_llms::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; +use litellm_types::llms::openai::ChatMessage; +use serde_json::{Map, Value}; + +/// A `/chat/completions` call as it crosses into the core. +/// +/// `optional_params` arrives already mapped to the provider's own parameter +/// names by the host, exactly as the messages route receives an already +/// Anthropic-shaped body. The core owns the conversation translation, the +/// provider call, and the response normalization. +pub struct ChatCompletionsRequest<'a> { + pub model: &'a str, + pub messages: Value, + pub optional_params: Map, + pub api_key: Option<&'a str>, + pub api_base: Option<&'a str>, + pub custom_llm_provider: Option<&'a str>, + pub extra_headers: Option>, + pub timeout: Option, +} + +pub struct ResolvedChatCompletionsRequest<'a> { + pub model: String, + pub config: &'static dyn BaseConfig, + pub messages: Vec, + pub optional_params: Map, + pub api_key: Option<&'a str>, + pub api_base: Option<&'a str>, + pub extra_headers: Option>, + pub timeout: Option, +} + +pub struct ProviderChatCompletionsRequest { + pub model: String, + pub config: &'static dyn BaseConfig, + pub url: String, + pub body: Value, + pub upstream_headers: Vec<(String, String)>, + pub auth: ChatCompletionsAuth, + pub optional_params: Map, + pub timeout: Option, +} diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index 4ff4333c4ac..3d740e39677 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -1,6 +1,4 @@ pub const OPENAI_DEFAULT_API_BASE: &str = "https://api.openai.com"; -pub const OPENAI_RESPONSES_DEFAULT_API_BASE: &str = "https://api.openai.com/v1"; -pub const OPENAI_RESPONSES_PATH: &str = "/responses"; /// Full-request timeout ceiling for Anthropic Messages provider calls, in /// seconds. Mirrors the Python Anthropic Messages default. The per-request @@ -10,19 +8,10 @@ pub(crate) const MESSAGES_TIMEOUT_SECS: u64 = 600; /// Connect timeout for Anthropic Messages provider calls, in seconds. pub(crate) const MESSAGES_CONNECT_TIMEOUT_SECS: u64 = 10; -/// Max characters of an upstream error body echoed across the call boundary -/// before truncation, so provider bodies are bounded and data-minimized. -pub(crate) const UPSTREAM_ERROR_BODY_MAX_CHARS: usize = 256; - /// Provider name used for Anthropic Messages when a deployment's provider model /// does not carry an explicit provider prefix. pub const ANTHROPIC_MESSAGES_PROVIDER: &str = "anthropic"; -/// Prefix identifying an Anthropic OAuth token. Mirrors Python's -/// `ANTHROPIC_OAUTH_TOKEN_PREFIX`, which is what makes `validate_environment` -/// authenticate with `authorization` and drop `x-api-key` entirely. -pub(crate) const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat"; - /// Full-request timeout ceiling for chat completions provider calls, in /// seconds. Mirrors the Python chat completions default. pub(crate) const CHAT_COMPLETIONS_TIMEOUT_SECS: u64 = 600; @@ -34,34 +23,3 @@ pub(crate) const AUDIO_TRANSCRIPTION_TIMEOUT_SECS: u64 = 600; /// `object` field every non-streaming chat completion response carries. pub const CHAT_COMPLETION_OBJECT: &str = "chat.completion"; - -/// Placeholder Python substitutes for empty or whitespace-only message text, -/// which Anthropic and Bedrock both reject. Must match -/// `_EMPTY_TEXT_PLACEHOLDER` in -/// `litellm/litellm_core_utils/prompt_templates/factory.py`. -pub const EMPTY_TEXT_PLACEHOLDER: &str = - "[System: Empty message content sanitised to satisfy protocol]"; - -pub(crate) const MEDIA_CONNECT_TIMEOUT_SECS: u64 = 10; - -pub(crate) const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024; -pub(crate) const OCR_HTTP_TIMEOUT_SECS: u64 = 600; -pub(crate) const OCR_CONNECT_TIMEOUT_SECS: u64 = 10; -pub const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024; -pub(crate) const OCR_DOWNLOAD_MAX_BYTES: u64 = 50 * 1024 * 1024; -pub(crate) const OCR_MAX_FETCH_REDIRECTS: usize = 10; -pub(crate) const OCR_POLL_TIMEOUT_SECS: u64 = 120; -pub(crate) const OCR_POLL_RETRY_SECS: u64 = 2; -pub(crate) const AZURE_DI_API_VERSION: &str = "2024-11-30"; -pub(crate) const AZURE_DI_SUBSCRIPTION_HEADER: &str = "Ocp-Apim-Subscription-Key"; -pub(crate) const AZURE_DI_DEFAULT_DPI: i64 = 96; -pub(crate) const AZURE_DI_DEFAULT_WIDTH: f64 = 8.5; -pub(crate) const AZURE_DI_DEFAULT_HEIGHT: f64 = 11.0; -pub(crate) const REDUCTO_API_BASE: &str = "https://platform.reducto.ai"; -pub(crate) const REDUCTO_API_KEY_ENV: &str = "REDUCTO_API_KEY"; -pub(crate) const REDUCTO_ID_PREFIX: &str = "reducto://"; -pub(crate) const AZURE_AI_OCR_PATH: &str = "/providers/mistral/azure/ocr"; -pub(crate) const MISTRAL_OCR_API_BASE: &str = "https://api.mistral.ai/v1"; - -pub(crate) const COHERE_PARSE_API_BASE: &str = "https://api.cohere.com"; -pub(crate) const COHERE_API_KEY_ENV: &str = "COHERE_API_KEY"; diff --git a/litellm-rust/crates/core/src/error.rs b/litellm-rust/crates/core/src/error.rs index 15d27602052..eb4cd2367ec 100644 --- a/litellm-rust/crates/core/src/error.rs +++ b/litellm-rust/crates/core/src/error.rs @@ -1,7 +1,9 @@ +use litellm_llms::base_llm::ocr::error::Error as OcrError; + #[derive(Debug, thiserror::Error)] pub enum Error { #[error(transparent)] - Ocr(#[from] crate::ocr::Error), + Ocr(#[from] OcrError), #[error(transparent)] Messages(#[from] crate::messages::Error), #[error(transparent)] diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index b1474f3f6c4..58aef6cd629 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -1,19 +1,10 @@ pub mod audio_transcription; -pub mod call_arguments; pub mod chat_completions; pub mod constants; pub mod error; -pub mod http_utils; -pub mod litellm_core_utils; -pub mod llms; pub mod machine; -mod media; pub mod messages; pub mod ocr; -pub mod params; pub mod responses; -mod serde_compat; -pub mod transport; -mod url_utils; pub use error::Error; diff --git a/litellm-rust/crates/core/src/litellm_core_utils/mod.rs b/litellm-rust/crates/core/src/litellm_core_utils/mod.rs deleted file mode 100644 index 7e3b3e96dda..00000000000 --- a/litellm-rust/crates/core/src/litellm_core_utils/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod get_llm_provider_logic; diff --git a/litellm-rust/crates/core/src/llms/anthropic/chat/mod.rs b/litellm-rust/crates/core/src/llms/anthropic/chat/mod.rs deleted file mode 100644 index 7bf4fc46291..00000000000 --- a/litellm-rust/crates/core/src/llms/anthropic/chat/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod streaming; diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/mod.rs b/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/mod.rs deleted file mode 100644 index 42d4fcdde0f..00000000000 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod batches; -pub mod count_tokens; -pub mod streaming; diff --git a/litellm-rust/crates/core/src/llms/anthropic/mod.rs b/litellm-rust/crates/core/src/llms/anthropic/mod.rs deleted file mode 100644 index 4943d80a45c..00000000000 --- a/litellm-rust/crates/core/src/llms/anthropic/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -pub mod chat; -pub mod experimental_pass_through; diff --git a/litellm-rust/crates/core/src/llms/azure_ai/mod.rs b/litellm-rust/crates/core/src/llms/azure_ai/mod.rs deleted file mode 100644 index 079e0c41eae..00000000000 --- a/litellm-rust/crates/core/src/llms/azure_ai/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod ocr; diff --git a/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/mod.rs b/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/mod.rs deleted file mode 100644 index 080f0a1183f..00000000000 --- a/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod transformation; diff --git a/litellm-rust/crates/core/src/llms/azure_ai/ocr/mod.rs b/litellm-rust/crates/core/src/llms/azure_ai/ocr/mod.rs deleted file mode 100644 index e106f50b0a7..00000000000 --- a/litellm-rust/crates/core/src/llms/azure_ai/ocr/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub(crate) mod cohere_parse_transformation; -pub(crate) mod common_utils; -pub(crate) mod document_intelligence; -pub(crate) mod transformation; diff --git a/litellm-rust/crates/core/src/llms/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/llms/azure_ai/ocr/transformation.rs deleted file mode 100644 index c6480ca6dac..00000000000 --- a/litellm-rust/crates/core/src/llms/azure_ai/ocr/transformation.rs +++ /dev/null @@ -1,615 +0,0 @@ -use litellm_auth::{InputSource, Sourced}; -use litellm_auth_azure::AzureAuthInputs; -use serde_json::Value; - -use crate::call_arguments::CallArguments; -use crate::constants::AZURE_AI_OCR_PATH; -use crate::llms::base_llm::ocr::transformation::{BaseOcrConfig, OcrRequestContext}; -use crate::llms::mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest}; -use crate::ocr::OcrClient; -use crate::ocr::document::{inline_remote_document, validate_inline_document}; -use crate::ocr::prepare::credential_env; -use crate::ocr::types::{LiteLLMOcrResponse, OcrConnection, OcrDocument, PreparedOcrRequest}; -use crate::params::OpaqueParams; -use crate::url_utils::ApiUrl; - -const AZURE_AI_API_KEY_ENV: &str = "AZURE_AI_API_KEY"; -const AZURE_AI_API_BASE_ENV: &str = "AZURE_AI_API_BASE"; - -#[derive(Clone, Debug, Default)] -pub(crate) struct AzureAiOcrConfig; - -impl BaseOcrConfig for AzureAiOcrConfig { - type OcrParams = OpaqueParams; - type ProviderRequest = MistralOcrRequest; - type Environment = Vec<(String, String)>; - - fn get_supported_ocr_params(&self, model: &str) -> &'static [&'static str] { - MistralOcrConfig.get_supported_ocr_params(model) - } - - fn get_api_key_env_var(&self) -> Option<&'static str> { - Some(AZURE_AI_API_KEY_ENV) - } - - fn map_ocr_params( - &self, - non_default_params: &CallArguments, - model: &str, - ) -> Result { - MistralOcrConfig.map_ocr_params(non_default_params, model) - } - - async fn validate_environment( - &self, - request: &PreparedOcrRequest, - _client: &OcrClient, - ) -> Result { - let config = AzureAuthInputs { - azure_ad_token_provider: request.azure_ad_token_provider.clone(), - ..AzureAuthInputs::from_sourced_optional_params( - &request.optional_params, - &request.input_sources, - )? - }; - self.resolve_headers(&request.connection, &config, &credential_env) - .await - } - - fn get_complete_url( - &self, - request: &PreparedOcrRequest, - _optional_params: &Self::OcrParams, - _environment: &Self::Environment, - ) -> Result { - self.build_ocr_url(request.connection.api_base.as_deref(), &credential_env) - } - - fn transform_ocr_request( - &self, - model: &str, - document: OcrDocument, - optional_params: &OpaqueParams, - headers: &[(String, String)], - ) -> Result { - MistralOcrConfig.transform_ocr_request(model, document, optional_params, headers) - } - - async fn async_transform_ocr_request( - &self, - model: &str, - document: OcrDocument, - optional_params: &OpaqueParams, - headers: &[(String, String)], - context: OcrRequestContext<'_>, - ) -> Result { - let document = inline_remote_document( - context.client.document_fetcher(), - document, - context.connection, - ) - .await?; - self.transform_ocr_request(model, document, optional_params, headers) - } - - fn transform_ocr_response( - &self, - model: &str, - raw_response: &[u8], - request_format: crate::ocr::types::OcrResponseFormat, - ) -> Result { - MistralOcrConfig.transform_ocr_response(model, raw_response, request_format) - } - - fn validate_request_body(&self, body: &Value) -> Result<(), crate::ocr::Error> { - validate_inline_document(&crate::ocr::prepare::body_document(body)?) - } -} - -impl AzureAiOcrConfig { - /// Python `AzureAIOCRConfig.validate_environment` requires the endpoint - /// before it resolves credentials; keep that order so a missing base is - /// reported without invoking any token provider. - pub(super) fn resolve_api_base( - api_base: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - nonblank(api_base.map(str::to_string)) - .or_else(|| nonblank(env_lookup(AZURE_AI_API_BASE_ENV))) - .ok_or(crate::ocr::Error::Auth( - litellm_auth::Error::MissingApiBase { - provider: "Azure AI", - environment_variable: AZURE_AI_API_BASE_ENV, - }, - )) - } - - async fn resolve_headers( - &self, - connection: &OcrConnection, - config: &AzureAuthInputs, - env_lookup: &(dyn Fn(&str) -> Option + Sync), - ) -> Result, crate::ocr::Error> { - Self::resolve_api_base(connection.api_base.as_deref(), env_lookup)?; - if crate::http_utils::has_header(&connection.extra_headers, "authorization") { - if config.azure_ad_token_provider.is_some() { - super::common_utils::resolve_entra(config, env_lookup).await?; - } - super::common_utils::validate_destination(connection, connection.extra_headers_source)?; - return Ok(connection.extra_headers.clone()); - } - let key = nonblank(connection.api_key.clone()) - .map(|value| Sourced::new(value, connection.api_key_source)) - .or_else(|| { - nonblank(self.get_api_key_env_var().and_then(env_lookup)) - .map(|value| Sourced::new(value, InputSource::Environment)) - }); - if let Some(key) = key { - super::common_utils::validate_destination(connection, key.source())?; - return Ok(bearer_headers(connection, key.value())); - } - let key = super::common_utils::resolve_entra(config, env_lookup) - .await? - .ok_or(crate::ocr::Error::MissingAzureAiCredentials)?; - super::common_utils::validate_destination(connection, key.source())?; - Ok(bearer_headers(connection, key.value())) - } - - fn build_ocr_url( - &self, - api_base: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, - ) -> Result { - let base = Self::resolve_api_base(api_base, env_lookup)?; - let path: Vec<&str> = AZURE_AI_OCR_PATH.trim_matches('/').split('/').collect(); - ApiUrl::parse(&base) - .and_then(|url| url.complete_path(&path)) - .map(|url| url.into_string()) - .map_err(|_| crate::ocr::Error::RequestField { - path: "api_base".into(), - }) - } -} - -fn bearer_headers(connection: &OcrConnection, key: &str) -> Vec<(String, String)> { - std::iter::once(("Authorization".into(), format!("Bearer {key}"))) - .chain(connection.extra_headers.clone()) - .collect() -} - -fn nonblank(value: Option) -> Option { - value - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) -} - -#[cfg(test)] -mod tests { - use rstest::{fixture, rstest}; - - use super::*; - - #[fixture] - fn connection() -> OcrConnection { - OcrConnection { - api_key: Some("request-key".into()), - api_base: Some("https://example.com".into()), - ..Default::default() - } - } - - #[rstest] - #[case::base_with_query( - "https://example.com/?tenant=a", - "https://example.com/providers/mistral/azure/ocr?tenant=a" - )] - #[case::complete_endpoint( - "https://example.com/providers/mistral/azure/ocr", - "https://example.com/providers/mistral/azure/ocr" - )] - fn completes_azure_path_and_preserves_query(#[case] api_base: &str, #[case] expected: &str) { - assert_eq!( - AzureAiOcrConfig - .build_ocr_url(Some(api_base), &|_| None) - .unwrap(), - expected - ); - } - - #[test] - fn missing_api_base_is_structured() { - assert!(matches!( - AzureAiOcrConfig::resolve_api_base(None, &|_| None), - Err(crate::ocr::Error::Auth( - litellm_auth::Error::MissingApiBase { - provider: "Azure AI", - environment_variable: AZURE_AI_API_BASE_ENV, - } - )) - )); - } - - #[rstest] - #[tokio::test] - async fn supplied_authorization_precedes_keys(connection: OcrConnection) { - let connection = OcrConnection { - extra_headers: vec![("authorization".into(), "Bearer prepared".into())], - ..connection - }; - assert_eq!( - AzureAiOcrConfig - .resolve_headers(&connection, &Default::default(), &|_| { - Some("environment-key".into()) - }) - .await - .unwrap(), - connection.extra_headers - ); - } - - #[rstest] - #[tokio::test] - async fn request_key_precedes_environment_key(connection: OcrConnection) { - assert_eq!( - AzureAiOcrConfig - .resolve_headers(&connection, &Default::default(), &|_| { - Some("environment-key".into()) - }) - .await - .unwrap()[0], - ("Authorization".into(), "Bearer request-key".into()) - ); - } - - #[tokio::test] - async fn request_endpoint_cannot_receive_environment_key() { - let connection = OcrConnection { - api_base: Some("https://request.example".into()), - api_base_source: InputSource::Request, - ..Default::default() - }; - - let error = AzureAiOcrConfig - .resolve_headers(&connection, &Default::default(), &|name| { - (name == AZURE_AI_API_KEY_ENV).then(|| "environment-key".into()) - }) - .await - .unwrap_err(); - - assert!( - error - .to_string() - .contains("request-controlled Azure endpoint") - ); - } - - #[tokio::test] - async fn request_endpoint_accepts_request_owned_key() { - let connection = OcrConnection { - api_key: Some("request-key".into()), - api_key_source: InputSource::Request, - api_base: Some("https://request.example".into()), - api_base_source: InputSource::Request, - ..Default::default() - }; - - let headers = AzureAiOcrConfig - .resolve_headers(&connection, &Default::default(), &|_| None) - .await - .unwrap(); - - assert_eq!( - headers[0], - ("Authorization".into(), "Bearer request-key".into()) - ); - } - - use serde_json::json; - - use crate::ocr::LocalOcrHost; - use crate::ocr::test_support::{ - MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request, - }; - - #[tokio::test] - async fn facade_executes_azure_mistral_with_prepared_auth() { - let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ - "pages":[{"index":0,"markdown":"hello"}], - "usage_info":{"pages_processed":1} - }))]) - .await; - let mut request = wire_request( - "azure_ai/model", - &base, - json!({"include_image_base64":true}), - ); - request.credentials.api_key = None; - request.transport.extra_headers = vec![( - "Authorization".into(), - "Bearer python-prepared-token".into(), - )]; - - let result = perform_ocr(request).await.unwrap(); - server.await.unwrap(); - assert_eq!(result.pages[0].markdown, "hello"); - let requests = seen.lock().unwrap(); - assert_eq!(requests.len(), 1); - assert!(requests[0].starts_with("POST /providers/mistral/azure/ocr ")); - assert!( - requests[0] - .to_ascii_lowercase() - .contains("authorization: bearer python-prepared-token\r\n") - ); - let body: Value = - serde_json::from_str(requests[0].split_once("\r\n\r\n").unwrap().1).unwrap(); - assert_eq!( - body, - json!({ - "model":"model", - "document":{"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}, - "include_image_base64":true - }) - ); - } - - #[tokio::test] - async fn facade_acquires_supplied_entra_token_for_final_request() { - let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; - let mut request = wire_request( - "azure_ai/model", - &base, - json!({"azure_ad_token":"rust-owned-token"}), - ); - request.credentials.api_key = None; - - perform_ocr(request).await.unwrap(); - server.await.unwrap(); - - let requests = seen.lock().unwrap(); - assert_eq!(requests.len(), 1); - assert!( - requests[0] - .to_ascii_lowercase() - .contains("authorization: bearer rust-owned-token\r\n") - ); - } - - #[tokio::test] - async fn rejects_non_inline_body_after_guardrails() { - let request = wire_request("azure_ai/model", "http://127.0.0.1:1", json!({})); - let host = LocalOcrHost::new(request).with_before_send(|mut wire, _| { - wire.body["document"] = json!({ - "type":"document_url", - "document_url":"https://example.com/not-inline.pdf" - }); - Ok(wire) - }); - let error = perform_ocr_with(host).await.unwrap_err(); - assert!(error.to_string().contains("data URI")); - } - - use std::sync::Arc; - use std::sync::atomic::{AtomicUsize, Ordering}; - - use litellm_auth::{ - ResolvedCredential, SecretValue, TokenFuture, TokenProvider, TokenProviderHandle, - }; - - use crate::ocr::LiteLLMOcrRequest; - use crate::ocr::test_support::header; - use crate::ocr::wire::decode_request; - - #[derive(Debug)] - struct CountingToken { - token: fn(usize) -> String, - calls: AtomicUsize, - } - - impl CountingToken { - fn new(token: fn(usize) -> String) -> Arc { - Arc::new(Self { - token, - calls: AtomicUsize::new(0), - }) - } - - fn calls(&self) -> usize { - self.calls.load(Ordering::SeqCst) - } - } - - impl TokenProvider for CountingToken { - fn acquire(&self) -> TokenFuture<'_> { - let call = self.calls.fetch_add(1, Ordering::SeqCst) + 1; - let token = SecretValue::new((self.token)(call)); - Box::pin(async move { - Ok(ResolvedCredential::AccessToken { - token, - expires_on: None, - }) - }) - } - } - - fn numbered_token(call: usize) -> String { - format!("callback-{call}") - } - - fn azure_request( - provider: &Arc, - api_base: Option<&str>, - api_key: Option<&str>, - extra_headers: Value, - optional_params: Value, - ) -> LiteLLMOcrRequest { - let wire = serde_json::from_value(json!({ - "model": "azure_ai/mistral-ocr-latest", - "document": {"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}, - "api_key": api_key, - "api_base": api_base, - "custom_llm_provider": null, - "extra_headers": extra_headers, - "optional_params": optional_params, - "timeout_seconds": 2.0 - })) - .unwrap(); - LiteLLMOcrRequest { - azure_ad_token_provider: Some(TokenProviderHandle::new(provider.clone())), - ..decode_request(wire).unwrap() - } - } - - fn ocr_page() -> MockResponse { - MockResponse::json(json!({"pages":[{"index":0,"markdown":"hello"}]})) - } - - #[tokio::test] - async fn token_provider_result_is_the_bearer_and_is_acquired_for_each_request() { - let provider = CountingToken::new(numbered_token); - let (base, seen, server) = mock_server(vec![ocr_page(), ocr_page()]).await; - - for _ in 0..2 { - perform_ocr(azure_request( - &provider, - Some(&base), - None, - Value::Null, - json!({}), - )) - .await - .unwrap(); - } - server.await.unwrap(); - - assert_eq!(provider.calls(), 2); - let requests = seen.lock().unwrap(); - assert_eq!( - requests - .iter() - .map(|request| header(request, "authorization")) - .collect::>(), - [Some("Bearer callback-1"), Some("Bearer callback-2")] - ); - } - - #[rstest] - #[case::api_key_skips_provider(Some("resource-key"), Value::Null, json!({}), "Bearer resource-key", 0)] - #[case::provider_beats_static_token( - None, - Value::Null, - json!({"azure_ad_token":"static-token"}), - "Bearer callback-1", - 1 - )] - #[case::header_wins_on_the_wire_but_provider_still_runs( - None, - json!({"Authorization":"Bearer override"}), - json!({}), - "Bearer override", - 1 - )] - #[tokio::test] - async fn credential_precedence( - #[case] api_key: Option<&str>, - #[case] extra_headers: Value, - #[case] optional_params: Value, - #[case] expected_authorization: &str, - #[case] expected_calls: usize, - ) { - let provider = CountingToken::new(numbered_token); - let (base, seen, server) = mock_server(vec![ocr_page()]).await; - - perform_ocr(azure_request( - &provider, - Some(&base), - api_key, - extra_headers, - optional_params, - )) - .await - .unwrap(); - server.await.unwrap(); - - assert_eq!(provider.calls(), expected_calls); - let requests = seen.lock().unwrap(); - assert_eq!(requests.len(), 1); - assert_eq!( - header(&requests[0], "authorization"), - Some(expected_authorization) - ); - } - - #[rstest] - #[case::missing_api_base( - false, - json!({}), - numbered_token, - |error: &crate::ocr::Error| matches!(error, crate::ocr::Error::Auth(litellm_auth::Error::MissingApiBase { - provider: "Azure AI", - environment_variable: AZURE_AI_API_BASE_ENV, - })), - 0 - )] - #[case::unsupported_oidc_reference( - true, - json!({"azure_ad_token":"oidc/assertion","client_id":"client","tenant_id":"tenant"}), - numbered_token, - |error: &crate::ocr::Error| matches!(error, crate::ocr::Error::Auth(litellm_auth::Error::UnsupportedOidcReference)), - 0 - )] - #[case::empty_provider_token_ignores_static_token( - true, - json!({"azure_ad_token":"static-token"}), - |_| String::new(), - |error: &crate::ocr::Error| matches!(error, crate::ocr::Error::MissingAzureAiCredentials), - 1 - )] - #[tokio::test] - async fn credential_failures_send_no_provider_request( - #[case] with_api_base: bool, - #[case] optional_params: Value, - #[case] token: fn(usize) -> String, - #[case] expected: fn(&crate::ocr::Error) -> bool, - #[case] expected_calls: usize, - ) { - let provider = CountingToken::new(token); - let (base, seen, server) = mock_server(vec![ocr_page()]).await; - - let error = perform_ocr(azure_request( - &provider, - with_api_base.then_some(base.as_str()), - None, - Value::Null, - optional_params, - )) - .await - .unwrap_err(); - server.abort(); - - assert!(expected(&error), "unexpected error: {error:?}"); - assert_eq!(provider.calls(), expected_calls); - assert!(seen.lock().unwrap().is_empty()); - } - - #[tokio::test] - async fn environment_supplies_api_base_and_bearer_key() { - let env = |name: &str| match name { - AZURE_AI_API_BASE_ENV => Some("https://env.example".to_string()), - AZURE_AI_API_KEY_ENV => Some("env-key".to_string()), - _ => None, - }; - let connection = OcrConnection::default(); - - let headers = AzureAiOcrConfig - .resolve_headers(&connection, &Default::default(), &env) - .await - .unwrap(); - let url = AzureAiOcrConfig.build_ocr_url(None, &env).unwrap(); - - assert_eq!( - headers, - [("Authorization".to_string(), "Bearer env-key".to_string())] - ); - assert_eq!(url, "https://env.example/providers/mistral/azure/ocr"); - } -} diff --git a/litellm-rust/crates/core/src/llms/base_llm/mod.rs b/litellm-rust/crates/core/src/llms/base_llm/mod.rs deleted file mode 100644 index 079e0c41eae..00000000000 --- a/litellm-rust/crates/core/src/llms/base_llm/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod ocr; diff --git a/litellm-rust/crates/core/src/llms/base_llm/ocr/mod.rs b/litellm-rust/crates/core/src/llms/base_llm/ocr/mod.rs deleted file mode 100644 index 080f0a1183f..00000000000 --- a/litellm-rust/crates/core/src/llms/base_llm/ocr/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod transformation; diff --git a/litellm-rust/crates/core/src/llms/base_llm/ocr/transformation.rs b/litellm-rust/crates/core/src/llms/base_llm/ocr/transformation.rs deleted file mode 100644 index b9dca3c9bd4..00000000000 --- a/litellm-rust/crates/core/src/llms/base_llm/ocr/transformation.rs +++ /dev/null @@ -1,213 +0,0 @@ -use std::future::Future; - -use serde::{Serialize, de::DeserializeOwned}; -use serde_json::Value; - -use crate::{ - call_arguments::CallArguments, - ocr::{ - OcrClient, - route::OcrHost, - types::{ - LiteLLMOcrResponse, OcrConnection, OcrCredentialInputs, OcrDocument, OcrResponseFormat, - PreparedOcrRequest, ResolvedOcrCredentials, - }, - }, -}; - -const HEALTH_CHECK_PDF_DATA_URI: &str = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="; - -/// Output of `validate_environment`: whatever a provider resolves up front -/// (headers at minimum; Vertex also carries the project id). -pub(crate) trait OcrEnvironment: Send + Sync { - fn headers(&self) -> &[(String, String)]; -} - -impl OcrEnvironment for Vec<(String, String)> { - fn headers(&self) -> &[(String, String)] { - self - } -} - -#[derive(Clone, Copy)] -pub(crate) struct OcrRequestContext<'a> { - pub client: &'a OcrClient, - pub connection: &'a OcrConnection, -} - -#[derive(Clone, Copy)] -pub(crate) struct OcrResponseContext<'a> { - pub client: &'a OcrClient, - pub connection: &'a OcrConnection, - pub host: &'a OcrHost, - pub request_format: OcrResponseFormat, - pub url: &'a str, - pub headers: &'a [(String, String)], -} - -pub(crate) trait BaseOcrConfig: Send + Sync + Sized + 'static { - type OcrParams: Send + Sync; - type ProviderRequest: Serialize + Send; - type Environment: OcrEnvironment; - - fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] { - &[] - } - - fn get_api_key_env_var(&self) -> Option<&'static str> { - None - } - - fn resolve_connection_params(&self, inputs: OcrCredentialInputs) -> ResolvedOcrCredentials { - ResolvedOcrCredentials { - api_key: inputs - .dynamic_api_key - .filter(|value| !value.value().is_empty()) - .or(inputs.api_key), - api_base: inputs - .dynamic_api_base - .filter(|value| !value.value().is_empty()) - .or(inputs.api_base), - } - } - - fn get_health_check_document(&self) -> OcrDocument { - OcrDocument::DocumentUrl { - document_url: HEALTH_CHECK_PDF_DATA_URI.into(), - extra_fields: Default::default(), - } - } - - fn map_ocr_params( - &self, - non_default_params: &CallArguments, - model: &str, - ) -> Result; - - fn validate_environment( - &self, - request: &PreparedOcrRequest, - client: &OcrClient, - ) -> impl Future> + Send; - - fn get_complete_url( - &self, - request: &PreparedOcrRequest, - optional_params: &Self::OcrParams, - environment: &Self::Environment, - ) -> Result; - - fn transform_ocr_request( - &self, - model: &str, - document: OcrDocument, - optional_params: &Self::OcrParams, - headers: &[(String, String)], - ) -> Result; - - fn async_transform_ocr_request( - &self, - model: &str, - document: OcrDocument, - optional_params: &Self::OcrParams, - headers: &[(String, String)], - _context: OcrRequestContext<'_>, - ) -> impl Future> + Send { - async move { self.transform_ocr_request(model, document, optional_params, headers) } - } - - fn transform_ocr_response( - &self, - model: &str, - raw_response: &[u8], - request_format: OcrResponseFormat, - ) -> Result; - - fn async_transform_ocr_response( - &self, - model: &str, - raw_response: reqwest::Response, - context: OcrResponseContext<'_>, - ) -> impl Future> + Send { - async move { - let bytes = crate::ocr::client::read_response_bytes( - raw_response, - context.connection.max_response_bytes, - ) - .await?; - crate::ocr::handler::emit_response_received(context.host, &bytes).await?; - self.transform_ocr_response(model, &bytes, context.request_format) - } - } - - fn get_error_class( - &self, - error_message: String, - status_code: u16, - headers: Vec<(String, String)>, - ) -> crate::ocr::Error { - crate::ocr::Error::Provider { - status: status_code, - body: error_message, - headers, - } - } - - /// Provider-specific check applied to the composed body, both before and - /// after guardrail hooks. Defaults to accepting any body. - fn validate_request_body(&self, _body: &Value) -> Result<(), crate::ocr::Error> { - Ok(()) - } - - /// Rust counterpart of `BaseLLMHTTPHandler._async_prepare_ocr_request`: - /// map params, validate environment, build URL, transform, compose body. - fn prepare_request( - &self, - request: &PreparedOcrRequest, - client: &OcrClient, - ) -> impl Future> + Send { - async move { - let params = self.map_ocr_params(&request.optional_params, &request.model)?; - let environment = self.validate_environment(request, client).await?; - let url = self.get_complete_url(request, ¶ms, &environment)?; - let headers = environment.headers(); - let body = self - .async_transform_ocr_request( - &request.model, - request.document.clone(), - ¶ms, - headers, - OcrRequestContext { - client, - connection: &request.connection, - }, - ) - .await?; - crate::ocr::prepare::transform_request_body( - client, - request, - &url, - headers, - body, - |body| self.validate_request_body(body), - ) - .await - } - } -} - -pub(crate) fn decode_and_normalize_response( - model: &str, - raw_response: &[u8], - request_format: OcrResponseFormat, - normalize: impl FnOnce(&str, T) -> Result, -) -> Result { - let decoded = crate::ocr::json::decode_response( - raw_response, - request_format == OcrResponseFormat::Native, - )?; - Ok(LiteLLMOcrResponse { - provider_native_response: decoded.native, - ..normalize(model, decoded.data)? - }) -} diff --git a/litellm-rust/crates/core/src/llms/cohere/mod.rs b/litellm-rust/crates/core/src/llms/cohere/mod.rs deleted file mode 100644 index 079e0c41eae..00000000000 --- a/litellm-rust/crates/core/src/llms/cohere/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod ocr; diff --git a/litellm-rust/crates/core/src/llms/cohere/ocr/mod.rs b/litellm-rust/crates/core/src/llms/cohere/ocr/mod.rs deleted file mode 100644 index 9cbe4df56e5..00000000000 --- a/litellm-rust/crates/core/src/llms/cohere/ocr/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) mod transformation; - -pub(crate) use transformation::{CohereOptions, validate_document}; diff --git a/litellm-rust/crates/core/src/llms/mistral/mod.rs b/litellm-rust/crates/core/src/llms/mistral/mod.rs deleted file mode 100644 index 079e0c41eae..00000000000 --- a/litellm-rust/crates/core/src/llms/mistral/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod ocr; diff --git a/litellm-rust/crates/core/src/llms/mistral/ocr/mod.rs b/litellm-rust/crates/core/src/llms/mistral/ocr/mod.rs deleted file mode 100644 index 080f0a1183f..00000000000 --- a/litellm-rust/crates/core/src/llms/mistral/ocr/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod transformation; diff --git a/litellm-rust/crates/core/src/llms/mod.rs b/litellm-rust/crates/core/src/llms/mod.rs deleted file mode 100644 index 4b93a5f971c..00000000000 --- a/litellm-rust/crates/core/src/llms/mod.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub mod anthropic; -pub mod azure_ai; -pub mod base_llm; -pub(crate) mod cohere; -pub(crate) mod mistral; -pub mod openai; -pub(crate) mod reducto; -pub(crate) mod vertex_ai; diff --git a/litellm-rust/crates/core/src/llms/reducto/mod.rs b/litellm-rust/crates/core/src/llms/reducto/mod.rs deleted file mode 100644 index 079e0c41eae..00000000000 --- a/litellm-rust/crates/core/src/llms/reducto/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod ocr; diff --git a/litellm-rust/crates/core/src/llms/reducto/ocr/mod.rs b/litellm-rust/crates/core/src/llms/reducto/ocr/mod.rs deleted file mode 100644 index 080f0a1183f..00000000000 --- a/litellm-rust/crates/core/src/llms/reducto/ocr/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod transformation; diff --git a/litellm-rust/crates/core/src/llms/vertex_ai/mod.rs b/litellm-rust/crates/core/src/llms/vertex_ai/mod.rs deleted file mode 100644 index 079e0c41eae..00000000000 --- a/litellm-rust/crates/core/src/llms/vertex_ai/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod ocr; diff --git a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/mod.rs b/litellm-rust/crates/core/src/llms/vertex_ai/ocr/mod.rs deleted file mode 100644 index f894ec145f8..00000000000 --- a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub(crate) mod common_utils; -pub(crate) mod deepseek_transformation; -pub(crate) mod transformation; diff --git a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/llms/vertex_ai/ocr/transformation.rs deleted file mode 100644 index bd7c5da7632..00000000000 --- a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/transformation.rs +++ /dev/null @@ -1,416 +0,0 @@ -use litellm_auth_gcp::{self as vertex, VertexConfig}; -use serde_json::Value; - -use super::common_utils::validate_destination; -use crate::{ - call_arguments::CallArguments, - llms::{ - base_llm::ocr::transformation::{BaseOcrConfig, OcrEnvironment, OcrRequestContext}, - mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest}, - }, - ocr::{ - OcrClient, - document::{inline_remote_document, validate_inline_document}, - prepare::credential_env, - types::{LiteLLMOcrResponse, OcrConnection, OcrDocument, PreparedOcrRequest}, - }, - params::OpaqueParams, - url_utils::ApiUrl, -}; - -const DEFAULT_LOCATION: &str = "us-central1"; - -#[derive(Clone, Debug, Default)] -pub(crate) struct VertexAiOcrConfig; - -impl BaseOcrConfig for VertexAiOcrConfig { - type OcrParams = OpaqueParams; - type ProviderRequest = MistralOcrRequest; - type Environment = vertex::VertexEnvironment; - - fn get_supported_ocr_params(&self, model: &str) -> &'static [&'static str] { - MistralOcrConfig.get_supported_ocr_params(model) - } - - fn get_api_key_env_var(&self) -> Option<&'static str> { - Some("VERTEX_AI_API_KEY") - } - - fn map_ocr_params( - &self, - non_default_params: &CallArguments, - model: &str, - ) -> Result { - MistralOcrConfig.map_ocr_params(non_default_params, model) - } - - async fn validate_environment( - &self, - request: &PreparedOcrRequest, - client: &OcrClient, - ) -> Result { - let config = VertexConfig::from_sourced_optional_params( - &request.optional_params, - &request.input_sources, - )?; - self.resolve_environment(&request.connection, &config, client) - .await - } - - fn get_complete_url( - &self, - request: &PreparedOcrRequest, - _optional_params: &Self::OcrParams, - environment: &Self::Environment, - ) -> Result { - let config = VertexConfig::from_sourced_optional_params( - &request.optional_params, - &request.input_sources, - )?; - let location = vertex::get_vertex_ai_location(&config, &credential_env) - .unwrap_or_else(|| DEFAULT_LOCATION.to_string()); - self.build_ocr_url( - request.connection.api_base.as_deref(), - &environment.project_id, - &location, - &request.model, - ) - } - - fn transform_ocr_request( - &self, - model: &str, - document: OcrDocument, - optional_params: &OpaqueParams, - headers: &[(String, String)], - ) -> Result { - MistralOcrConfig.transform_ocr_request(model, document, optional_params, headers) - } - - async fn async_transform_ocr_request( - &self, - model: &str, - document: OcrDocument, - optional_params: &OpaqueParams, - headers: &[(String, String)], - context: OcrRequestContext<'_>, - ) -> Result { - let document = inline_remote_document( - context.client.document_fetcher(), - document, - context.connection, - ) - .await?; - self.transform_ocr_request(model, document, optional_params, headers) - } - - fn transform_ocr_response( - &self, - model: &str, - raw_response: &[u8], - request_format: crate::ocr::types::OcrResponseFormat, - ) -> Result { - MistralOcrConfig.transform_ocr_response(model, raw_response, request_format) - } - - fn validate_request_body(&self, body: &Value) -> Result<(), crate::ocr::Error> { - validate_inline_document(&crate::ocr::prepare::body_document(body)?) - } -} - -impl OcrEnvironment for vertex::VertexEnvironment { - fn headers(&self) -> &[(String, String)] { - &self.headers - } -} - -impl VertexAiOcrConfig { - async fn resolve_environment( - &self, - connection: &OcrConnection, - config: &VertexConfig, - client: &OcrClient, - ) -> Result { - validate_destination(connection)?; - client - .vertex_auth() - .validate_environment( - connection.extra_headers.clone(), - connection.api_key.as_deref(), - config, - &credential_env, - ) - .await - .map_err(crate::ocr::Error::from) - } - - fn build_ocr_url( - &self, - api_base: Option<&str>, - project: &str, - location: &str, - model: &str, - ) -> Result { - validate_location(location)?; - let default_base = format!("https://{location}-aiplatform.googleapis.com"); - let base = api_base - .map(str::trim) - .filter(|base| !base.is_empty()) - .unwrap_or(&default_base); - let prediction = format!("{model}:rawPredict"); - ApiUrl::parse(base) - .and_then(|url| { - url.complete_path(&[ - "v1", - "projects", - project, - "locations", - location, - "publishers", - "mistralai", - "models", - &prediction, - ]) - }) - .map(|url| url.into_string()) - .map_err(|_| crate::ocr::Error::RequestField { - path: "api_base".into(), - }) - } -} - -fn validate_location(location: &str) -> Result<(), crate::ocr::Error> { - let valid = !location.is_empty() - && location - .bytes() - .all(|value| value.is_ascii_lowercase() || value.is_ascii_digit() || value == b'-') - && location - .as_bytes() - .first() - .is_some_and(u8::is_ascii_alphanumeric) - && location - .as_bytes() - .last() - .is_some_and(u8::is_ascii_alphanumeric); - if valid { - return Ok(()); - } - Err(crate::ocr::Error::RequestField { - path: "vertex_location".into(), - }) -} - -#[cfg(test)] -mod tests { - use rstest::rstest; - - use super::VertexAiOcrConfig; - - #[test] - fn endpoint_uses_location_project_and_model() { - assert_eq!( - VertexAiOcrConfig - .build_ocr_url(None, "proj-1", "europe-west4", "mistral-ocr-maas") - .unwrap(), - "https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict" - ); - } - - #[test] - fn endpoint_rejects_invalid_location() { - assert!( - VertexAiOcrConfig - .build_ocr_url(None, "proj-1", "attacker.example/path", "model") - .is_err() - ); - } - - use litellm_auth::InputSource; - use serde_json::{Value, json}; - - use crate::ocr::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; - - fn request_body(request: &str) -> Value { - serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap() - } - - #[tokio::test] - async fn facade_executes_vertex_mistral_with_resolved_project_and_location() { - let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ - "pages":[{"index":0,"markdown":"hello"}], - "usage_info":{"pages_processed":1} - }))]) - .await; - let request = wire_request( - "vertex_ai/mistral-ocr-maas", - &base, - json!({ - "vertex_project":"project-1", - "vertex_location":"europe-west4", - "extract_footer":true - }), - ); - - let response = perform_ocr(request).await.unwrap(); - server.await.unwrap(); - assert_eq!(response.pages[0].markdown, "hello"); - let requests = seen.lock().unwrap(); - assert_eq!(requests.len(), 1); - assert!(requests[0].starts_with( - "POST /v1/projects/project-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict " - )); - assert!( - requests[0] - .to_ascii_lowercase() - .contains("authorization: bearer test-key") - ); - assert_eq!( - request_body(&requests[0]), - json!({ - "model":"mistral-ocr-maas", - "document":{"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}, - "extract_footer":true - }) - ); - } - - #[tokio::test] - async fn supplied_authorization_is_forwarded_without_a_static_token() { - let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; - let mut request = wire_request( - "vertex_ai/model", - &base, - json!({"vertex_project":"project-1"}), - ); - request.credentials.api_key = None; - request.transport.extra_headers = vec![("authorization".into(), "Bearer supplied".into())]; - - perform_ocr(request).await.unwrap(); - server.await.unwrap(); - assert!( - seen.lock().unwrap()[0] - .to_ascii_lowercase() - .contains("authorization: bearer supplied") - ); - } - - #[tokio::test] - async fn invalid_credentials_fail_before_provider_http() { - let request = wire_request( - "vertex_ai/model", - "http://127.0.0.1:1", - json!({"vertex_credentials": true}), - ); - let error = perform_ocr(request).await.unwrap_err(); - assert!(error.to_string().contains("vertex_credentials")); - } - - #[tokio::test] - async fn request_controlled_api_base_is_rejected_before_vertex_auth() { - let mut request = wire_request( - "vertex_ai/mistral-ocr-maas", - "https://caller.example", - json!({"vertex_project":"project-1"}), - ); - request.credentials.api_base = Some(litellm_auth::Sourced::new( - "https://caller.example".into(), - InputSource::Request, - )); - - let error = perform_ocr(request).await.unwrap_err(); - assert!( - error - .to_string() - .contains("request-controlled Vertex AI endpoint") - ); - } - - #[rstest] - #[case::mistral(false)] - #[case::vertex(true)] - #[tokio::test] - async fn configs_build_complete_requests_and_share_mistral_normalization( - #[case] use_vertex: bool, - ) { - use std::time::Duration; - - use crate::{ - llms::{ - base_llm::ocr::transformation::BaseOcrConfig, - mistral::ocr::transformation::MistralOcrConfig, - vertex_ai::ocr::transformation::VertexAiOcrConfig, - }, - ocr::test_support::ocr_client, - }; - - let client = ocr_client(); - let options = json!({ - "pages": [0, 2], - "include_image_base64": true, - "vertex_project": "project-1", - "vertex_location": "us-central1", - "unknown": "preserved" - }); - let direct = wire_request( - "mistral/mistral-ocr-maas", - "https://mistral.test", - options.clone(), - ); - let vertex = wire_request("vertex_ai/mistral-ocr-maas", "https://vertex.test", options); - let direct = crate::ocr::prepare::prepare_request_for_test( - crate::ocr::test_support::resolved_request(direct), - ); - let vertex = crate::ocr::prepare::prepare_request_for_test( - crate::ocr::test_support::resolved_request(vertex), - ); - let direct_http = MistralOcrConfig - .prepare_request(&direct, &client) - .await - .unwrap(); - let vertex_http = VertexAiOcrConfig - .prepare_request(&vertex, &client) - .await - .unwrap(); - assert_eq!(direct_http.url().as_str(), "https://mistral.test/v1/ocr"); - assert_eq!( - vertex_http.url().as_str(), - "https://vertex.test/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict" - ); - let http = if use_vertex { - &vertex_http - } else { - &direct_http - }; - assert_eq!(http.method(), reqwest::Method::POST); - assert_eq!(http.headers()["authorization"], "Bearer test-key"); - assert_eq!(http.headers()["content-type"], "application/json"); - assert_eq!(http.timeout(), Some(&Duration::from_secs(2))); - let body: Value = serde_json::from_slice(http.body().unwrap().as_bytes().unwrap()).unwrap(); - assert_eq!( - body, - json!({ - "model": "mistral-ocr-maas", - "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, - "pages": [0, 2], - "include_image_base64": true, - "unknown": "preserved" - }) - ); - let payload = serde_json::to_vec( - &json!({"pages": [{"index": 0, "markdown": "hello"}], "extra": "preserved"}), - ) - .unwrap(); - let direct_response = MistralOcrConfig - .transform_ocr_response(&direct.model, &payload, Default::default()) - .unwrap() - .into_json(); - let vertex_response = VertexAiOcrConfig - .transform_ocr_response(&vertex.model, &payload, Default::default()) - .unwrap() - .into_json(); - assert_eq!(direct_response, vertex_response); - assert_eq!(direct_response["model"], "mistral-ocr-maas"); - assert_eq!(direct_response["object"], "ocr"); - assert_eq!(direct_response["extra"], "preserved"); - } -} diff --git a/litellm-rust/crates/core/src/machine/mod.rs b/litellm-rust/crates/core/src/machine/mod.rs index f4ca3e407e8..279a2d65c97 100644 --- a/litellm-rust/crates/core/src/machine/mod.rs +++ b/litellm-rust/crates/core/src/machine/mod.rs @@ -37,7 +37,7 @@ struct PendingOp { /// The provider side of the machine: how the in-flight call reaches its host. pub struct HostChannel { - ops: Option>>, + ops: mpsc::UnboundedSender>, } impl Clone for HostChannel { @@ -48,24 +48,14 @@ impl Clone for HostChannel { } } -impl HostChannel { - /// A channel with no host behind it: the wire request goes out unchanged, events go - /// nowhere, and route operations fail. For tests that prepare a request without - /// driving it. - #[cfg(test)] - pub(crate) fn detached() -> Self { - Self { ops: None } - } -} - impl HostChannel where R::Error: From, { async fn invoke(&self, op: HostOp) -> Result, R::Error> { - let ops = self.ops.as_ref().ok_or(MachineFault::Abandoned)?; let (reply, answer) = oneshot::channel(); - ops.send(PendingOp { op, reply }) + self.ops + .send(PendingOp { op, reply }) .map_err(|_| MachineFault::Abandoned)?; answer.await.map_err(|_| MachineFault::Abandoned.into()) } @@ -82,9 +72,6 @@ where wire: WireRequest, context: RequestContext, ) -> Result { - if self.ops.is_none() { - return Ok(wire); - } let op = HostOp::BeforeSend { wire: Box::new(wire), context: Box::new(context), @@ -96,9 +83,6 @@ where } pub async fn emit(&self, event: CallEvent) -> Result<(), R::Error> { - if self.ops.is_none() { - return Ok(()); - } match self.invoke(HostOp::Emit(event)).await? { HostResult::Emitted => Ok(()), _ => Err(MachineFault::Mismatch.into()), @@ -128,7 +112,7 @@ where Self { execution: Execution::Unstarted(Box::new(execute)), ops, - channel: HostChannel { ops: Some(ops_tx) }, + channel: HostChannel { ops: ops_tx }, reply: None, } } diff --git a/litellm-rust/crates/core/src/messages/common_utils.rs b/litellm-rust/crates/core/src/messages/common_utils.rs index 81d67520abe..ec392324784 100644 --- a/litellm-rust/crates/core/src/messages/common_utils.rs +++ b/litellm-rust/crates/core/src/messages/common_utils.rs @@ -1,13 +1,15 @@ -use litellm_providers::{ +pub(super) use litellm_llms::custom_httpx::http_handler::{ + has_bearer_auth, has_header, truncate_error_body, +}; +use litellm_llms::{ anthropic::experimental_pass_through::messages::transformation::ANTHROPIC_MESSAGES_CONFIG, azure_ai::anthropic::messages_transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG, base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig, + custom_httpx::http_handler::string_headers as shared_string_headers, }; use serde_json::{Map, Value}; use super::Error; -use crate::http_utils::string_headers as shared_string_headers; -pub(super) use crate::http_utils::{has_bearer_auth, has_header, truncate_error_body}; const HEADER_CONTEXT: &str = "messages"; diff --git a/litellm-rust/crates/core/src/messages/error.rs b/litellm-rust/crates/core/src/messages/error.rs index cdb4de4645f..71bb748c50d 100644 --- a/litellm-rust/crates/core/src/messages/error.rs +++ b/litellm-rust/crates/core/src/messages/error.rs @@ -1,3 +1,5 @@ +use litellm_llms::base_llm::chat::transformation::Error as LlmError; + #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { #[error("invalid provider: {0}")] @@ -13,33 +15,20 @@ pub enum Error { #[error(transparent)] Auth(#[from] litellm_auth::Error), #[error(transparent)] - Transport(#[from] crate::transport::Error), + Transport(#[from] litellm_llms::custom_httpx::transport::Error), #[error(transparent)] - Headers(#[from] crate::http_utils::HeaderError), - #[error("stream framing failed: {0}")] - StreamFraming(String), - #[error("Anthropic SSE frame has no data")] - MissingStreamData, - #[error("Anthropic stream event is invalid: {0}")] - InvalidStreamEvent(String), - #[error("Bedrock event payload is invalid: {0}")] - InvalidBedrockPayload(String), - #[error("Bedrock event payload has invalid base64: {0}")] - InvalidBedrockBase64(String), + Headers(#[from] litellm_llms::custom_httpx::http_handler::HeaderError), } -impl From for Error { - fn from(error: litellm_providers::messages::Error) -> Self { +impl From for Error { + fn from(error: LlmError) -> Self { match error { - litellm_providers::messages::Error::MissingField(field) => Self::MissingField(field), - litellm_providers::messages::Error::InvalidRequest(message) => { - Self::InvalidRequest(message) - } - litellm_providers::messages::Error::InvalidResponse(message) => { - Self::InvalidResponse(message) - } - litellm_providers::messages::Error::Unsupported(reason) => Self::Unsupported(reason), - litellm_providers::messages::Error::Auth(error) => Self::Auth(error), + error @ LlmError::InvalidType { .. } => Self::InvalidRequest(error.to_string()), + LlmError::MissingField(field) => Self::MissingField(field), + LlmError::InvalidRequest(message) => Self::InvalidRequest(message), + LlmError::InvalidResponse(message) => Self::InvalidResponse(message), + LlmError::Unsupported(reason) => Self::Unsupported(reason), + LlmError::Auth(error) => Self::Auth(error), } } } @@ -58,14 +47,6 @@ impl Error { } pub fn is_response(&self) -> bool { - matches!( - self, - Self::InvalidResponse(_) - | Self::StreamFraming(_) - | Self::MissingStreamData - | Self::InvalidStreamEvent(_) - | Self::InvalidBedrockPayload(_) - | Self::InvalidBedrockBase64(_) - ) + matches!(self, Self::InvalidResponse(_)) } } diff --git a/litellm-rust/crates/core/src/messages/handler.rs b/litellm-rust/crates/core/src/messages/handler.rs index ff3ae5765ff..b95402b1a7a 100644 --- a/litellm-rust/crates/core/src/messages/handler.rs +++ b/litellm-rust/crates/core/src/messages/handler.rs @@ -1,11 +1,11 @@ +use litellm_llms::custom_httpx::http_handler::http_request; +use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; + use super::{ - Error, - client::http_client, - common_utils::truncate_error_body, + Error, client::http_client, common_utils::truncate_error_body, prepare::prepare_provider_request, - types::{AnthropicMessagesResponse, MessagesRequest}, }; -use crate::{constants::ANTHROPIC_MESSAGES_PROVIDER, http_utils::http_request}; +use crate::{constants::ANTHROPIC_MESSAGES_PROVIDER, messages::types::MessagesRequest}; pub(super) async fn execute_messages_provider_call( request: MessagesRequest<'_>, @@ -19,21 +19,26 @@ pub(super) async fn execute_messages_provider_call( request_builder = request_builder.timeout(duration); } - let response = http_request(request_builder) - .await - .map_err(|err| Error::Transport(crate::transport::Error::Network(err.to_string())))?; + let response = http_request(request_builder).await.map_err(|err| { + Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( + err.to_string(), + )) + })?; let status = response.status(); - let text = response - .text() - .await - .map_err(|err| Error::Transport(crate::transport::Error::Network(err.to_string())))?; + let text = response.text().await.map_err(|err| { + Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( + err.to_string(), + )) + })?; if !status.is_success() { - return Err(Error::Transport(crate::transport::Error::Http { - status: status.as_u16(), - body: truncate_error_body(&text), - })); + return Err(Error::Transport( + litellm_llms::custom_httpx::transport::Error::Http { + status: status.as_u16(), + body: truncate_error_body(&text), + }, + )); } let response = serde_json::from_str(&text) @@ -60,19 +65,24 @@ pub(super) async fn execute_messages_provider_stream( request_builder = request_builder.timeout(duration); } - let response = http_request(request_builder) - .await - .map_err(|err| Error::Transport(crate::transport::Error::Network(err.to_string())))?; + let response = http_request(request_builder).await.map_err(|err| { + Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( + err.to_string(), + )) + })?; let status = response.status(); if !status.is_success() { - let text = response - .text() - .await - .map_err(|err| Error::Transport(crate::transport::Error::Network(err.to_string())))?; - return Err(Error::Transport(crate::transport::Error::Http { - status: status.as_u16(), - body: truncate_error_body(&text), - })); + let text = response.text().await.map_err(|err| { + Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( + err.to_string(), + )) + })?; + return Err(Error::Transport( + litellm_llms::custom_httpx::transport::Error::Http { + status: status.as_u16(), + body: truncate_error_body(&text), + }, + )); } Ok(response) } diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index 812094f637c..c3d7bea48ff 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -8,14 +8,16 @@ //! can splice the event stream to its own caller. mod error; +pub mod types; pub use error::Error; mod client; mod common_utils; mod handler; mod prepare; use handler::{execute_messages_provider_call, execute_messages_provider_stream}; -pub use litellm_providers::messages::types; -use types::{AnthropicMessagesResponse, MessagesRequest}; +use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; + +use crate::messages::types::MessagesRequest; pub async fn messages(request: MessagesRequest<'_>) -> Result { execute_messages_provider_call(request).await diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index 4a6c871172f..8b676803871 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,4 +1,5 @@ -use litellm_providers::base_llm::anthropic_messages::transformation::{ +use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; +use litellm_llms::base_llm::anthropic_messages::transformation::{ BaseAnthropicMessagesConfig, MessagesAuthStrategy, }; use serde_json::{Map, Value}; @@ -6,11 +7,8 @@ use serde_json::{Map, Value}; use super::{ Error, common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers}, - types::{MessagesRequest, ProviderMessagesRequest}, -}; -use crate::litellm_core_utils::get_llm_provider_logic::{ - CustomLlmProvider, get_custom_llm_provider, }; +use crate::messages::types::{MessagesRequest, ProviderMessagesRequest}; pub(super) fn prepare_provider_request( request: MessagesRequest<'_>, diff --git a/litellm-rust/crates/core/src/messages/tests.rs b/litellm-rust/crates/core/src/messages/tests.rs index 98b9bd626a9..55d8ead8e8b 100644 --- a/litellm-rust/crates/core/src/messages/tests.rs +++ b/litellm-rust/crates/core/src/messages/tests.rs @@ -12,8 +12,8 @@ use super::{ has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body, }, messages, - types::MessagesRequest, }; +use crate::messages::types::MessagesRequest; async fn read_http_request(socket: &mut TcpStream) -> String { let mut request = Vec::new(); @@ -82,7 +82,7 @@ fn string_headers_rejects_non_string_values() { let err = string_headers(Some(headers)).expect_err("non-string header rejected"); assert_eq!( err, - Error::Headers(crate::http_utils::HeaderError { + Error::Headers(litellm_llms::custom_httpx::http_handler::HeaderError { context: "messages", name: "x-count".to_string(), actual: "number", @@ -432,7 +432,7 @@ async fn messages_maps_provider_error_status_to_http_error() { assert!(matches!( err, - Error::Transport(crate::transport::Error::Http { status: 401, .. }) + Error::Transport(litellm_llms::custom_httpx::transport::Error::Http { status: 401, .. }) )); } diff --git a/litellm-rust/crates/core/src/messages/types.rs b/litellm-rust/crates/core/src/messages/types.rs new file mode 100644 index 00000000000..a73ceffad7a --- /dev/null +++ b/litellm-rust/crates/core/src/messages/types.rs @@ -0,0 +1,24 @@ +use std::time::Duration; + +use litellm_llms::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig; +use serde_json::{Map, Value}; + +pub struct MessagesRequest<'a> { + pub model: &'a str, + pub body: Value, + pub api_key: Option<&'a str>, + pub api_base: Option<&'a str>, + pub custom_llm_provider: Option<&'a str>, + pub extra_headers: Option>, + pub timeout: Option, +} + +pub struct ProviderMessagesRequest { + pub provider: String, + pub model: String, + pub config: &'static dyn BaseAnthropicMessagesConfig, + pub url: String, + pub body: Value, + pub upstream_headers: Vec<(String, String)>, + pub timeout: Option, +} diff --git a/litellm-rust/crates/core/src/ocr/arguments.rs b/litellm-rust/crates/core/src/ocr/arguments.rs index 2b27496fb5f..43f1c6d6d43 100644 --- a/litellm-rust/crates/core/src/ocr/arguments.rs +++ b/litellm-rust/crates/core/src/ocr/arguments.rs @@ -1,5 +1,7 @@ +use litellm_core_utils::call_arguments::ArgumentSpec; +use litellm_llms::base_llm::ocr::error::Error; + use super::provider_config::{OcrConfigKind, resolve_provider_config}; -use crate::call_arguments::ArgumentSpec; const COMMON_OPTION_FIELDS: &[&str] = &["req_format", "extra_body", "max_response_bytes"]; const AZURE_AUTH_OPTION_FIELDS: &[&str] = &[ @@ -29,7 +31,7 @@ pub fn is_supported_request(model: &str, custom_llm_provider: Option<&str>) -> b pub fn consumed_optional_param_names( model: &str, custom_llm_provider: Option<&str>, -) -> Result, super::Error> { +) -> Result, Error> { let (model, config) = resolve_provider_config(model, custom_llm_provider)?; let provider_fields = config.get_supported_ocr_params(&model); let auth_fields: &[&str] = match config { @@ -61,7 +63,7 @@ pub(crate) fn is_secret_param(name: &str) -> bool { pub fn consumed_optional_params( model: &str, custom_llm_provider: Option<&str>, -) -> Result, super::Error> { +) -> Result, Error> { consumed_optional_param_names(model, custom_llm_provider).map(|names| { names .into_iter() diff --git a/litellm-rust/crates/core/src/ocr/client.rs b/litellm-rust/crates/core/src/ocr/client.rs index bc8094953cf..03782d91f24 100644 --- a/litellm-rust/crates/core/src/ocr/client.rs +++ b/litellm-rust/crates/core/src/ocr/client.rs @@ -1,176 +1,20 @@ -use std::{sync::OnceLock, time::Duration}; - -use bytes::{Bytes, BytesMut}; -use litellm_auth_gcp::VertexAuth; -use serde::de::DeserializeOwned; - -use super::{ - json::{DecodedOcrResponse, decode_response}, - types::{LiteLLMOcrRequest, LiteLLMOcrResponse}, +use litellm_llms::{ + base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse}, + custom_httpx::llm_http_handler::OcrClient, }; -use crate::{constants::OCR_CONNECT_TIMEOUT_SECS, media::MediaFetcher}; -#[derive(Clone)] -pub struct OcrClient { - provider_http: reqwest::Client, - polling_http: reqwest::Client, - document_fetcher: MediaFetcher, - vertex_auth: VertexAuth, +use crate::ocr::{ + route::{LocalOcrHost, ocr_machine}, + types::LiteLLMOcrRequest, +}; + +pub async fn perform( + client: &OcrClient, + request: LiteLLMOcrRequest, +) -> Result { + litellm_callbacks::run::run(ocr_machine(client.clone()), &LocalOcrHost::new(request)).await } -impl OcrClient { - pub fn new(provider_http: reqwest::Client) -> Result { - let document_fetcher = MediaFetcher::new().map_err(crate::transport::Error::from)?; - Ok(Self { - provider_http, - polling_http: no_redirect_http()?, - document_fetcher, - vertex_auth: VertexAuth::default(), - }) - } - - pub fn shared() -> Result { - shared_client() - } - - pub async fn perform( - &self, - request: LiteLLMOcrRequest, - ) -> Result { - litellm_callbacks::run::run( - super::ocr_machine(self.clone()), - &super::LocalOcrHost::new(request), - ) - .await - } - - pub(crate) fn provider_http(&self) -> &reqwest::Client { - &self.provider_http - } - - pub(crate) fn polling_http(&self) -> &reqwest::Client { - &self.polling_http - } - - pub(crate) fn document_fetcher(&self) -> &MediaFetcher { - &self.document_fetcher - } - - pub(crate) fn vertex_auth(&self) -> &VertexAuth { - &self.vertex_auth - } - - #[cfg(test)] - pub(crate) fn for_test(provider_http: reqwest::Client, document_http: reqwest::Client) -> Self { - Self { - provider_http, - polling_http: no_redirect_http().expect("test polling client builds"), - document_fetcher: MediaFetcher::for_test(document_http), - vertex_auth: VertexAuth::default(), - } - } -} - -fn no_redirect_http() -> Result { - reqwest::Client::builder() - .connect_timeout(Duration::from_secs(OCR_CONNECT_TIMEOUT_SECS)) - .redirect(reqwest::redirect::Policy::none()) - .build() - .map_err(crate::transport::Error::from) -} - -pub(crate) fn shared_client() -> Result { - static CLIENT: OnceLock> = OnceLock::new(); - let client = CLIENT - .get_or_init(|| { - reqwest::Client::builder() - .connect_timeout(Duration::from_secs(OCR_CONNECT_TIMEOUT_SECS)) - .build() - .map_err(crate::transport::Error::from) - .and_then(OcrClient::new) - }) - .clone()?; - Ok(client) -} - -pub async fn ocr(request: LiteLLMOcrRequest) -> Result { - shared_client()?.perform(request).await -} - -pub async fn read_json_response( - response: reqwest::Response, - native: bool, - max_response_bytes: usize, -) -> Result, crate::ocr::Error> { - let bytes = read_response_bytes(response, max_response_bytes).await?; - decode_response(&bytes, native) -} - -pub(crate) async fn read_response_bytes( - mut response: reqwest::Response, - limit: usize, -) -> Result { - let status = response.status(); - if status.is_success() - && response - .content_length() - .is_some_and(|length| length > limit as u64) - { - return Err(crate::ocr::Error::TooLarge { limit }); - } - let mut bytes = BytesMut::new(); - while let Some(chunk) = response.chunk().await.map_err(transport_error)? { - let remaining = limit.saturating_sub(bytes.len()); - if status.is_success() && chunk.len() > remaining { - return Err(crate::ocr::Error::TooLarge { limit }); - } - bytes.extend_from_slice(&chunk[..chunk.len().min(remaining)]); - if !status.is_success() && bytes.len() == limit { - break; - } - } - if !status.is_success() { - return Err(crate::transport::Error::Http { - status: status.as_u16(), - body: String::from_utf8_lossy(&bytes).into_owned(), - } - .into()); - } - Ok(bytes.freeze()) -} - -pub(crate) fn transport_error(error: reqwest::Error) -> crate::ocr::Error { - if error.is_timeout() { - return crate::ocr::Error::Transport(crate::transport::Error::Http { - status: 408, - body: "OCR request timed out".into(), - }); - } - crate::transport::Error::from(error).into() -} - -#[cfg(test)] -mod tests { - use super::*; - - #[tokio::test] - async fn request_timeout_has_an_http_408_status() { - let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { - let _connection = listener.accept().await.unwrap(); - tokio::time::sleep(Duration::from_secs(1)).await; - }); - let error = reqwest::Client::new() - .get(format!("http://{address}")) - .timeout(Duration::from_millis(10)) - .send() - .await - .unwrap_err(); - assert!(matches!( - transport_error(error), - crate::ocr::Error::Transport(crate::transport::Error::Http { status: 408, .. }) - )); - server.abort(); - } +pub async fn ocr(request: LiteLLMOcrRequest) -> Result { + perform(&OcrClient::shared()?, request).await } diff --git a/litellm-rust/crates/core/src/ocr/document.rs b/litellm-rust/crates/core/src/ocr/document.rs index a3515627dd7..2b89421373f 100644 --- a/litellm-rust/crates/core/src/ocr/document.rs +++ b/litellm-rust/crates/core/src/ocr/document.rs @@ -1,20 +1,14 @@ use std::{collections::BTreeMap as Map, io::Read, path::Path}; use base64::{Engine, engine::general_purpose::STANDARD}; -use data_url::{DataUrl, DataUrlError, forgiving_base64::DecodeError, mime::Mime}; -use reqwest::Url; - -use super::{ - Error as OcrError, Error as OcrRequestError, Error as OcrResponseError, - types::{OcrConnection, OcrDocument, OcrDocumentInput}, -}; -use crate::{ - constants::{OCR_INLINE_MAX_BYTES, OCR_MAX_FETCH_REDIRECTS}, - media::{DownloadPolicy, Error as MediaError, MediaFetcher}, - transport::Error as TransportError, +use litellm_llms::base_llm::ocr::{ + error::Error, + transformation::{OCR_INLINE_MAX_BYTES, OcrDocument}, }; -pub fn prepare_document(input: OcrDocumentInput) -> Result { +use crate::ocr::types::OcrDocumentInput; + +pub fn prepare_document(input: OcrDocumentInput) -> Result { match input { OcrDocumentInput::Document(document) => Ok(document), OcrDocumentInput::Path { path, mime_type } => { @@ -29,23 +23,20 @@ pub fn prepare_document(input: OcrDocumentInput) -> Result Err(super::Error::InvalidRequest( + OcrDocumentInput::HostReader { .. } => Err(Error::InvalidRequest( "OCR file reader was not read by the host".into(), )), } } -pub fn read_path_document( - path: &Path, - mime_type: Option<&str>, -) -> Result { +pub fn read_path_document(path: &Path, mime_type: Option<&str>) -> Result { let mut bytes = Vec::new(); std::fs::File::open(path) .and_then(|file| { file.take(OCR_INLINE_MAX_BYTES as u64 + 1) .read_to_end(&mut bytes) }) - .map_err(|source| super::Error::FileRead { + .map_err(|source| Error::FileRead { path: path.to_owned(), source: std::sync::Arc::new(source), })?; @@ -57,17 +48,17 @@ pub fn encode_file_document( bytes: &[u8], file_name: Option<&str>, mime_type: Option<&str>, -) -> Result { +) -> Result { if bytes.is_empty() { - return Err(OcrRequestError::EmptyFile); + return Err(Error::EmptyFile); } if bytes.len() > OCR_INLINE_MAX_BYTES { - return Err(OcrRequestError::InlineDocumentTooLarge); + return Err(Error::InlineDocumentTooLarge); } if let Some(value) = mime_type && !valid_mime_type(value) { - return Err(OcrRequestError::InvalidMimeType(value.into())); + return Err(Error::InvalidMimeType(value.into())); } let mime_type = mime_type .map(str::to_string) @@ -117,105 +108,12 @@ pub fn mime_type_for_name(name: &str) -> &'static str { } } -pub(crate) struct InlineDocument<'a>(DataUrl<'a>); - -impl<'a> InlineDocument<'a> { - pub(crate) fn parse(source: &'a str) -> Result, OcrRequestError> { - match DataUrl::process(source) { - Ok(url) => Ok(Some(Self(url))), - Err(DataUrlError::NotADataUrl) => Ok(None), - Err(DataUrlError::NoComma) => Err(OcrRequestError::InvalidDataUri), - } - } - - pub(crate) fn mime_type(&self) -> &Mime { - self.0.mime_type() - } - - pub(crate) fn decode(&self, max_bytes: usize) -> Result, OcrRequestError> { - let mut body = Vec::new(); - self.0 - .decode(|bytes| { - if bytes.len() > max_bytes.saturating_sub(body.len()) { - return Err(OcrRequestError::InlineDocumentTooLarge); - } - body.extend_from_slice(bytes); - Ok(()) - }) - .map_err(|error| match error { - DecodeError::InvalidBase64(_) => OcrRequestError::InvalidDataUri, - DecodeError::WriteError(error) => error, - })?; - Ok(body) - } -} - -pub(crate) fn validate_inline_document(document: &OcrDocument) -> Result<(), OcrRequestError> { - let inline = - InlineDocument::parse(document.source())?.ok_or(OcrRequestError::InvalidDataUri)?; - inline.decode(crate::constants::OCR_INLINE_MAX_BYTES)?; - Ok(()) -} - -pub(crate) async fn inline_remote_document( - fetcher: &MediaFetcher, - document: OcrDocument, - connection: &OcrConnection, -) -> Result { - let source = document.source(); - if !document.is_remote() { - validate_inline_document(&document)?; - return Ok(document); - } - let url = Url::parse(source).map_err(|_| OcrRequestError::RequestField { - path: "document URL".into(), - })?; - let downloaded = fetcher - .fetch( - url, - DownloadPolicy { - timeout: connection.timeout, - max_bytes: connection.max_download_bytes, - max_redirects: OCR_MAX_FETCH_REDIRECTS, - }, - ) - .await - .map_err(map_media_error)?; - let result = document.with_source(format!( - "data:{};base64,{}", - downloaded.content_type, - STANDARD.encode(downloaded.bytes) - )); - validate_inline_document(&result)?; - Ok(result) -} - -fn map_media_error(error: MediaError) -> OcrError { - match error { - MediaError::BlockedUrl => OcrRequestError::BlockedDocumentUrl, - MediaError::DownloadDisabled => OcrRequestError::DownloadDisabled, - MediaError::DownloadTooLarge => OcrRequestError::DownloadTooLarge, - MediaError::TooManyRedirects => OcrRequestError::TooManyRedirects, - MediaError::MissingRedirectLocation => OcrResponseError::MissingRedirectLocation, - MediaError::InvalidRedirect => OcrResponseError::InvalidRedirect, - MediaError::Http(status) => TransportError::Http { - status, - body: "OCR document download failed".into(), - } - .into(), - MediaError::Timeout => TransportError::Http { - status: 408, - body: "OCR document download timed out".into(), - } - .into(), - MediaError::Transport(error) => error.into(), - } -} - #[cfg(test)] mod tests { use std::collections::BTreeMap as Map; + use litellm_llms::base_llm::ocr::document::InlineDocument; + use super::*; fn document(source: &str) -> OcrDocument { @@ -291,12 +189,12 @@ mod tests { path: path.clone(), mime_type: None, }), - Err(OcrRequestError::InlineDocumentTooLarge) + Err(Error::InlineDocumentTooLarge) )); std::fs::remove_dir_all(&dir).unwrap(); let missing = dir.join("missing.pdf"); - let Err(super::super::Error::FileRead { path, source, .. }) = + let Err(super::Error::FileRead { path, source, .. }) = prepare_document(OcrDocumentInput::Path { path: missing.clone(), mime_type: None, @@ -327,7 +225,7 @@ mod tests { let bytes = vec![b'a'; OCR_INLINE_MAX_BYTES + 1]; assert!(matches!( encode_file_document(&bytes, None, None), - Err(OcrRequestError::InlineDocumentTooLarge) + Err(Error::InlineDocumentTooLarge) )); let document = encode_file_document(&bytes[..OCR_INLINE_MAX_BYTES], None, None).unwrap(); let inline = InlineDocument::parse(document.source()).unwrap().unwrap(); @@ -349,102 +247,4 @@ mod tests { assert!(encode_file_document(b"abc", None, Some(mime)).is_err()); } } - - #[test] - fn decodes_data_urls_and_limits_decoded_size() { - for (source, expected) in [ - ("data:application/pdf;base64,YWJj", b"abc".as_slice()), - ("DATA:application/pdf;BASE64,YWI", b"ab".as_slice()), - ("data:,a%20b%00%FF", b"a b\0\xff".as_slice()), - ] { - let inline = InlineDocument::parse(source).unwrap().unwrap(); - assert_eq!(inline.decode(expected.len()).unwrap(), expected); - assert!(matches!( - inline.decode(expected.len() - 1), - Err(OcrRequestError::InlineDocumentTooLarge) - )); - } - } - - #[test] - fn preserves_mime_parameters_and_standard_default() { - let inline = InlineDocument::parse("data:application/pdf;version=1.7;base64,YQ==") - .unwrap() - .unwrap(); - assert!(inline.mime_type().matches("application", "pdf")); - assert_eq!(inline.mime_type().get_parameter("version"), Some("1.7")); - let default = InlineDocument::parse("data:,a").unwrap().unwrap(); - assert!(default.mime_type().matches("text", "plain")); - assert_eq!( - default.mime_type().get_parameter("charset"), - Some("US-ASCII") - ); - } - - #[test] - fn rejects_invalid_inline_documents() { - for source in [ - "https://example.com/document.pdf", - "data:application/pdf;base64", - "data:application/pdf;base64,INVALID!", - ] { - assert!(validate_inline_document(&document(source)).is_err()); - } - } - - #[tokio::test] - async fn remote_conversion_preserves_kind_and_isolates_provider_credentials() { - use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::TcpListener, - }; - - let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); - let address = listener.local_addr().unwrap(); - let server = tokio::spawn(async move { - let (mut socket, _) = listener.accept().await.unwrap(); - let mut request = vec![0_u8; 2048]; - let count = socket.read(&mut request).await.unwrap(); - socket - .write_all(b"HTTP/1.1 200 OK\r\nContent-Type: image/png; charset=binary\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc") - .await - .unwrap(); - String::from_utf8_lossy(&request[..count]).into_owned() - }); - let mut provider_headers = reqwest::header::HeaderMap::new(); - provider_headers.insert( - reqwest::header::AUTHORIZATION, - reqwest::header::HeaderValue::from_static("Bearer provider-secret"), - ); - let provider_http = reqwest::Client::builder() - .default_headers(provider_headers) - .build() - .unwrap(); - let document_http = reqwest::Client::builder() - .redirect(reqwest::redirect::Policy::none()) - .build() - .unwrap(); - let client = super::super::OcrClient::for_test(provider_http, document_http); - let converted = inline_remote_document( - client.document_fetcher(), - OcrDocument::ImageUrl { - image_url: format!("http://{address}/image"), - extra_fields: Map::from_iter([("detail".into(), Some("high".into()))]), - }, - &OcrConnection::default(), - ) - .await - .unwrap(); - let request = server.await.unwrap(); - - assert_eq!( - converted, - OcrDocument::ImageUrl { - image_url: "data:image/png;base64,YWJj".into(), - extra_fields: Map::from_iter([("detail".into(), Some("high".into()))]), - } - ); - assert!(!request.to_ascii_lowercase().contains("authorization")); - assert!(!request.contains("provider-secret")); - } } diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/core/src/ocr/handler.rs index 450ac91f55d..33cb8a8d32a 100644 --- a/litellm-rust/crates/core/src/ocr/handler.rs +++ b/litellm-rust/crates/core/src/ocr/handler.rs @@ -1,117 +1,81 @@ -use litellm_callbacks::event::{CallEvent, RawResponse}; +use futures_util::future::BoxFuture; +use litellm_callbacks::event::{CallEvent, Passthrough, RawResponse, RequestContext, WireRequest}; +use litellm_llms::{ + base_llm::ocr::{ + error::Error, + transformation::{LiteLLMOcrResponse, PreparedOcrRequest}, + }, + custom_httpx::llm_http_handler::{CallHooks, OcrClient}, +}; +use serde_json::Value; use super::{ - OcrClient, + arguments::is_secret_param, prepare::prepare_request, provider_config::OcrConfigKind, route::OcrHost, - types::{LiteLLMOcrResponse, PreparedOcrRequest, ResolvedOcrRequest}, }; -use crate::llms::base_llm::ocr::transformation::OcrResponseContext; +use crate::ocr::types::ResolvedOcrRequest; pub(crate) async fn perform_ocr_request( client: &OcrClient, request: ResolvedOcrRequest, host: &OcrHost, caller_document: bool, -) -> Result { +) -> Result { request.response_format()?; - PreparedOcrCall::prepare(client.clone(), request, host, caller_document) - .await? - .execute() - .await + let config = request.config; + let request = prepare_request(request, caller_document); + let hooks = OcrCallHooks::new(host.clone(), &request, config); + config.ocr(client, &request, &hooks).await } -pub(crate) struct PreparedOcrCall { - client: OcrClient, - request: PreparedOcrRequest, - http: reqwest::Request, +/// Lets provider code reach the host mid-call, filling in the request context only the +/// route knows. +pub(crate) struct OcrCallHooks { + host: OcrHost, + model: String, + custom_llm_provider: &'static str, + optional_params: Value, + secret_fields: Vec, } -impl PreparedOcrCall { - pub(crate) async fn prepare( - client: OcrClient, - request: ResolvedOcrRequest, - host: &OcrHost, - caller_document: bool, - ) -> Result { - let request = super::prepare::prepare_request(request, host.clone(), caller_document); - let http = request.config.prepare_request(&request, &client).await?; - Ok(Self { - client, - request, - http, - }) - } - - pub(crate) async fn execute(self) -> Result { - let url = self.http.url().to_string(); - let headers = request_headers(&self.http)?; - let response = - crate::http_utils::execute_http_request(self.client.provider_http(), self.http) - .await - .map_err(super::client::transport_error)?; - if !response.status().is_success() { - let headers = response - .headers() - .iter() - .filter_map(|(name, value)| { - value - .to_str() - .ok() - .map(|value| (name.to_string(), value.to_string())) - }) - .collect(); - return match super::client::read_response_bytes( - response, - self.request.connection.max_response_bytes, - ) - .await - { - Err(super::Error::Transport(crate::transport::Error::Http { status, body })) => { - Err(self.request.config.get_error_class(body, status, headers)) - } - Err(error) => Err(error), - Ok(_) => unreachable!("non-success response produces an HTTP error"), - }; +impl OcrCallHooks { + pub(crate) fn new(host: OcrHost, request: &PreparedOcrRequest, config: OcrConfigKind) -> Self { + Self { + host, + model: request.model.clone(), + custom_llm_provider: config.provider().into(), + optional_params: Value::Object(request.optional_params.clone().into()), + secret_fields: request + .optional_params + .keys() + .filter(|name| is_secret_param(name)) + .cloned() + .collect(), } - let model = &self.request.model; - let context = OcrResponseContext { - client: &self.client, - connection: &self.request.connection, - host: &self.request.host, - request_format: self.request.response_format()?, - url: &url, - headers: &headers, - }; - self.request - .config - .async_transform_ocr_response(model, response, context) - .await } } -fn request_headers(request: &reqwest::Request) -> Result, super::Error> { - request - .headers() - .iter() - .map(|(name, value)| { - value - .to_str() - .map(|value| (name.to_string(), value.to_string())) - .map_err(|_| super::Error::RequestField { - path: "headers".into(), - }) - }) - .collect() -} +impl CallHooks for OcrCallHooks { + fn before_send( + &self, + wire: WireRequest, + passthrough_fields: Passthrough, + ) -> BoxFuture<'_, Result> { + let context = RequestContext { + model: self.model.clone(), + custom_llm_provider: self.custom_llm_provider.into(), + optional_params: self.optional_params.clone(), + passthrough_fields, + secret_fields: self.secret_fields.clone(), + }; + Box::pin(self.host.before_send(wire, context)) + } -pub(crate) async fn emit_response_received( - host: &OcrHost, - bytes: &[u8], -) -> Result<(), super::Error> { - host.emit(CallEvent::ResponseReceived { - raw: RawResponse { - body: String::from_utf8_lossy(bytes).into_owned(), - }, - }) - .await + fn response_received<'a>(&'a self, body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> { + Box::pin(self.host.emit(CallEvent::ResponseReceived { + raw: RawResponse { + body: String::from_utf8_lossy(body).into_owned(), + }, + })) + } } diff --git a/litellm-rust/crates/core/src/ocr/json.rs b/litellm-rust/crates/core/src/ocr/json.rs deleted file mode 100644 index d4651838a2d..00000000000 --- a/litellm-rust/crates/core/src/ocr/json.rs +++ /dev/null @@ -1,62 +0,0 @@ -use serde::de::{DeserializeOwned, IntoDeserializer}; -use serde_json::{Map, Value}; - -#[derive(Debug)] -pub struct DecodedOcrResponse { - pub data: T, - pub native: Option>, - pub text: String, -} - -pub(crate) fn decode_request_value( - value: Value, - prefix: &str, -) -> Result { - serde_path_to_error::deserialize(value.into_deserializer()).map_err(|error| { - crate::ocr::Error::RequestField { - path: format!("{prefix}.{}", error.path()), - } - }) -} - -pub(crate) fn decode_response_value( - value: Value, - prefix: &str, -) -> Result { - serde_path_to_error::deserialize(value.into_deserializer()).map_err(|error| { - crate::ocr::Error::ResponseField { - path: format!("{prefix}.{}", error.path()), - } - }) -} - -pub(crate) fn decode_response( - bytes: &[u8], - native: bool, -) -> Result, crate::ocr::Error> { - let mut deserializer = serde_json::Deserializer::from_slice(bytes); - let data = serde_path_to_error::deserialize(&mut deserializer).map_err(|error| { - crate::ocr::Error::ResponseField { - path: error.path().to_string(), - } - })?; - deserializer - .end() - .map_err(|_| crate::ocr::Error::ResponseField { - path: "response".into(), - })?; - let native = if native { - Some( - serde_json::from_slice(bytes).map_err(|_| crate::ocr::Error::ResponseField { - path: "response".into(), - })?, - ) - } else { - None - }; - Ok(DecodedOcrResponse { - data, - native, - text: String::from_utf8_lossy(bytes).into_owned(), - }) -} diff --git a/litellm-rust/crates/core/src/ocr/mod.rs b/litellm-rust/crates/core/src/ocr/mod.rs index 75d85da7957..e7f77acc3f8 100644 --- a/litellm-rust/crates/core/src/ocr/mod.rs +++ b/litellm-rust/crates/core/src/ocr/mod.rs @@ -1,29 +1,13 @@ -mod arguments; +pub mod arguments; pub mod client; -pub(crate) mod document; -pub mod error; -pub use error::Error; +pub mod document; pub(crate) mod handler; -pub(crate) mod json; pub(crate) mod prepare; -mod provider_config; +pub mod provider_config; pub mod route; pub mod types; pub mod wire; -pub use arguments::{ - consumed_optional_param_names, consumed_optional_params, is_supported_request, -}; -pub use client::{OcrClient, ocr}; -pub use document::{encode_file_document, mime_type_for_name, read_path_document}; -pub use provider_config::{get_api_key_env_var, get_health_check_document}; -pub use route::{LocalOcrHost, Ocr, OcrHost, OcrMachine, OcrOp, OcrOpResult, ocr_machine}; -pub use types::{ - LiteLLMOcrRequest, LiteLLMOcrResponse, OcrConnection, OcrConnectionInputs, OcrCredentialInputs, - OcrDocument, OcrDocumentInput, OcrFileContent, OcrPage, OcrPageDimensions, OcrPageImage, - OcrTransportConfig, OcrUsageInfo, -}; - #[cfg(test)] #[path = "../../tests/azure_ai_ocr.rs"] mod azure_ai_tests; @@ -31,6 +15,9 @@ mod azure_ai_tests; #[path = "../../tests/azure_document_intelligence_ocr.rs"] mod azure_document_intelligence_tests; #[cfg(test)] +#[path = "../../tests/cohere_ocr.rs"] +mod cohere_tests; +#[cfg(test)] #[path = "../../tests/deepseek_ocr.rs"] mod deepseek_tests; #[cfg(test)] diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index 2de72660794..24c3f43e2b4 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -1,156 +1,20 @@ -use litellm_callbacks::event::{Passthrough, RequestContext, WireRequest}; -use serde::Serialize; -use serde_json::{Map, Value}; +use litellm_auth::{InputSource, Sourced}; +use litellm_llms::base_llm::ocr::transformation::{ + OcrConnection, OcrCredentialInputs, PreparedOcrRequest, credential_env, +}; -use super::OcrClient; -use super::route::OcrHost; -use super::types::{OcrConnection, OcrDocument, PreparedOcrRequest, ResolvedOcrRequest}; - -pub(crate) async fn transform_request_body( - client: &OcrClient, - request: &PreparedOcrRequest, - url: &str, - headers: &[(String, String)], - body: B, - validate: impl Fn(&Value) -> Result<(), super::Error>, -) -> Result -where - B: Serialize, -{ - let composed = crate::call_arguments::compose_body( - &request.optional_params, - &body, - request.config.get_supported_ocr_params(&request.model), - )?; - validate(&composed)?; - let passthrough_fields = Passthrough::unchanged(&caller_inputs(request)?, &composed); - let changed = request - .host - .before_send( - wire_request(url, headers, composed), - request_context(request, passthrough_fields), - ) - .await?; - if !changed.body.is_object() { - return Err(super::Error::RequestField { - path: "guardrail.body".into(), - }); - } - validate(&changed.body)?; - build_http_request(client, request, url, &changed.headers, &changed.body) -} - -fn wire_request(url: &str, headers: &[(String, String)], body: Value) -> WireRequest { - WireRequest { - url: url.into(), - headers: headers.to_vec(), - body, - } -} - -fn caller_inputs(request: &PreparedOcrRequest) -> Result, super::Error> { - let document = request - .caller_document - .then(|| serde_json::to_value(&request.document)) - .transpose() - .map_err(|_| super::Error::RequestField { - path: "document".into(), - })?; - let params: Map = request.optional_params.clone().into(); - Ok(params - .into_iter() - .chain(document.map(|document| ("document".to_string(), document))) - .collect()) -} - -fn request_context( - request: &PreparedOcrRequest, - passthrough_fields: Passthrough, -) -> RequestContext { - RequestContext { - model: request.model.clone(), - custom_llm_provider: request.provider_name().into(), - optional_params: Value::Object(request.optional_params.clone().into()), - passthrough_fields, - secret_fields: request - .optional_params - .keys() - .filter(|name| super::arguments::is_secret_param(name)) - .cloned() - .collect(), - } -} - -pub(crate) fn build_http_request( - client: &OcrClient, - request: &PreparedOcrRequest, - url: &str, - headers: &[(String, String)], - body: &B, -) -> Result { - let builder = client - .provider_http() - .post(url) - .json(body) - .timeout(request.connection.timeout); - crate::http_utils::with_headers(builder, headers, crate::http_utils::HeaderPolicy::All) - .build() - .map_err(crate::transport::Error::from) - .map_err(super::Error::from) -} - -pub(crate) async fn guardrail_document( - request: &PreparedOcrRequest, - url: &str, - headers: &[(String, String)], -) -> Result<(OcrDocument, Vec<(String, String)>), super::Error> { - let body = serde_json::to_value(&request.document).map_err(|_| super::Error::RequestField { - path: "document".into(), - })?; - let changed = request - .host - .before_send( - wire_request(url, headers, body), - request_context(request, Passthrough::default()), - ) - .await?; - let document = super::json::decode_request_value(changed.body, "guardrail.document")?; - Ok((document, changed.headers)) -} - -pub(crate) fn body_document(body: &Value) -> Result { - let document = body - .get("document") - .and_then(Value::as_object) - .ok_or_else(|| super::Error::RequestField { - path: "body.document".into(), - })?; - let source = document - .iter() - .filter(|(name, _)| matches!(name.as_str(), "type" | "image_url" | "document_url")) - .map(|(name, value)| (name.clone(), value.clone())) - .collect(); - super::json::decode_request_value(Value::Object(source), "body.document") -} - -pub(crate) fn credential_env(name: &str) -> Option { - std::env::var(name).ok() -} +use super::provider_config::OcrProvider; +use crate::ocr::types::{LiteLLMOcrRequest, ResolvedOcrRequest}; pub(crate) fn prepare_request( request: ResolvedOcrRequest, - host: OcrHost, caller_document: bool, ) -> PreparedOcrRequest { - use litellm_auth::{InputSource, Sourced}; - let credentials = request.credentials.clone(); let api_base_env = match request.config.provider() { - super::provider_config::OcrProvider::Mistral => Some("MISTRAL_API_BASE"), - super::provider_config::OcrProvider::AzureAi => Some("AZURE_AI_API_BASE"), - super::provider_config::OcrProvider::Cohere - | super::provider_config::OcrProvider::Reducto - | super::provider_config::OcrProvider::VertexAi => None, + OcrProvider::Mistral => Some("MISTRAL_API_BASE"), + OcrProvider::AzureAi => Some("AZURE_AI_API_BASE"), + OcrProvider::Cohere | OcrProvider::Reducto | OcrProvider::VertexAi => None, }; let dynamic_api_key = credentials.dynamic_api_key.or_else(|| { credentials.api_key.clone().or_else(|| { @@ -170,31 +34,41 @@ pub(crate) fn prepare_request( }); let resolved = request .config - .resolve_connection_params(super::types::OcrCredentialInputs { + .resolve_connection_params(OcrCredentialInputs { dynamic_api_key, dynamic_api_base, ..credentials }); - let transport = request.transport.clone(); - PreparedOcrRequest::new( - request, - OcrConnection::new(resolved, transport), - host, + let LiteLLMOcrRequest { + model, + document, + transport, + optional_params, + input_sources, + azure_ad_token_provider, + .. + } = request; + PreparedOcrRequest { + model, + document, + connection: OcrConnection::new(resolved, transport), caller_document, - ) + optional_params, + input_sources, + azure_ad_token_provider, + } } #[cfg(test)] pub(crate) fn prepare_request_for_test(request: ResolvedOcrRequest) -> PreparedOcrRequest { - prepare_request(request, OcrHost::detached(), true) + prepare_request(request, true) } #[cfg(test)] mod tests { + use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options}; use serde_json::json; - use crate::call_arguments::{CallArguments, compose_body, parse_options}; - #[derive(serde::Deserialize)] struct KnownParams { pages: Option>, diff --git a/litellm-rust/crates/core/src/ocr/provider_config.rs b/litellm-rust/crates/core/src/ocr/provider_config.rs index 0121f2dfdf4..d12b8cfee95 100644 --- a/litellm-rust/crates/core/src/ocr/provider_config.rs +++ b/litellm-rust/crates/core/src/ocr/provider_config.rs @@ -1,48 +1,66 @@ +use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; +use litellm_llms::{ + azure_ai::ocr::{ + cohere_parse_transformation::AzureAICohereParseConfig, + document_intelligence::transformation::AzureDocumentIntelligenceOcrConfig, + transformation::AzureAiOcrConfig, + }, + base_llm::ocr::{ + error::Error, + transformation::{ + BaseOcrConfig, LiteLLMOcrResponse, OcrCredentialInputs, OcrDocument, + PreparedOcrRequest, ResolvedOcrCredentials, + }, + }, + cohere::ocr::transformation::CohereParseConfig, + custom_httpx::llm_http_handler::{self, CallHooks, OcrClient}, + mistral::ocr::transformation::MistralOcrConfig, + reducto::ocr::transformation::{ReductoParseLegacyConfig, ReductoParseV3Config}, + vertex_ai::ocr::{ + deepseek_transformation::VertexAIDeepSeekOCRConfig, transformation::VertexAiOcrConfig, + }, +}; use strum::{EnumString, IntoStaticStr}; -use super::{ - OcrClient, - types::{ - LiteLLMOcrResponse, OcrCredentialInputs, OcrDocument, PreparedOcrRequest, - ResolvedOcrCredentials, - }, -}; -use crate::{ - litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}, - llms::{ - azure_ai::ocr::{ - cohere_parse_transformation::AzureAICohereParseConfig, - document_intelligence::transformation::AzureDocumentIntelligenceOcrConfig, - transformation::AzureAiOcrConfig, - }, - base_llm::ocr::transformation::{BaseOcrConfig, OcrResponseContext}, - cohere::ocr::transformation::CohereParseConfig, - mistral::ocr::transformation::MistralOcrConfig, - reducto::ocr::transformation::{ReductoParseLegacyConfig, ReductoParseV3Config}, - vertex_ai::ocr::{ - deepseek_transformation::VertexAIDeepSeekOCRConfig, transformation::VertexAiOcrConfig, - }, - }, -}; - -macro_rules! dispatch_config { - ($config:expr, $method:ident($($argument:expr),* $(,)?)) => { - dispatch_config!(@arms $config, $method($($argument),*), ) - }; - ($config:expr, $method:ident($($argument:expr),* $(,)?).await) => { - dispatch_config!(@arms $config, $method($($argument),*), .await) - }; - (@arms $config:expr, $method:ident($($argument:expr),*), $($suffix:tt)*) => { - match $config { - OcrConfigKind::Cohere => CohereParseConfig.$method($($argument),*)$($suffix)*, - OcrConfigKind::Mistral => MistralOcrConfig.$method($($argument),*)$($suffix)*, - OcrConfigKind::AzureAi => AzureAiOcrConfig.$method($($argument),*)$($suffix)*, - OcrConfigKind::AzureCohere => AzureAICohereParseConfig.$method($($argument),*)$($suffix)*, - OcrConfigKind::AzureDocumentIntelligence => AzureDocumentIntelligenceOcrConfig.$method($($argument),*)$($suffix)*, - OcrConfigKind::ReductoLegacy => ReductoParseLegacyConfig.$method($($argument),*)$($suffix)*, - OcrConfigKind::ReductoV3 => ReductoParseV3Config.$method($($argument),*)$($suffix)*, - OcrConfigKind::VertexAi => VertexAiOcrConfig.$method($($argument),*)$($suffix)*, - OcrConfigKind::VertexDeepSeek => VertexAIDeepSeekOCRConfig.$method($($argument),*)$($suffix)*, +macro_rules! with_config { + ($kind:expr, $config:ident => $body:expr) => { + match $kind { + OcrConfigKind::Cohere => { + let $config = CohereParseConfig; + $body + } + OcrConfigKind::Mistral => { + let $config = MistralOcrConfig; + $body + } + OcrConfigKind::AzureAi => { + let $config = AzureAiOcrConfig; + $body + } + OcrConfigKind::AzureCohere => { + let $config = AzureAICohereParseConfig; + $body + } + OcrConfigKind::AzureDocumentIntelligence => { + let $config = AzureDocumentIntelligenceOcrConfig; + $body + } + OcrConfigKind::ReductoLegacy => { + let $config = ReductoParseLegacyConfig; + $body + } + OcrConfigKind::ReductoV3 => { + let $config = ReductoParseV3Config; + $body + } + OcrConfigKind::VertexAi => { + let $config = VertexAiOcrConfig; + $body + } + OcrConfigKind::VertexDeepSeek => { + let $config = VertexAIDeepSeekOCRConfig; + $body + } } }; } @@ -74,58 +92,38 @@ impl OcrConfigKind { } pub(crate) fn get_supported_ocr_params(self, model: &str) -> &'static [&'static str] { - dispatch_config!(self, get_supported_ocr_params(model)) + with_config!(self, config => config.get_supported_ocr_params(model)) } pub(crate) fn get_api_key_env_var(self) -> Option<&'static str> { - dispatch_config!(self, get_api_key_env_var()) + with_config!(self, config => config.get_api_key_env_var()) } pub(crate) fn get_health_check_document(self) -> OcrDocument { - dispatch_config!(self, get_health_check_document()) + with_config!(self, config => config.get_health_check_document()) } pub(crate) fn resolve_connection_params( self, inputs: OcrCredentialInputs, ) -> ResolvedOcrCredentials { - dispatch_config!(self, resolve_connection_params(inputs)) + with_config!(self, config => config.resolve_connection_params(inputs)) } - pub(crate) fn get_error_class( + pub(crate) async fn ocr( self, - message: String, - status: u16, - headers: Vec<(String, String)>, - ) -> super::Error { - dispatch_config!(self, get_error_class(message, status, headers)) - } - - pub(crate) async fn prepare_request( - self, - request: &PreparedOcrRequest, client: &OcrClient, - ) -> Result { - dispatch_config!(self, prepare_request(request, client).await) - } - - pub(crate) async fn async_transform_ocr_response( - self, - model: &str, - raw_response: reqwest::Response, - context: OcrResponseContext<'_>, - ) -> Result { - dispatch_config!( - self, - async_transform_ocr_response(model, raw_response, context).await - ) + request: &PreparedOcrRequest, + hooks: &dyn CallHooks, + ) -> Result { + with_config!(self, config => llm_http_handler::ocr(&config, client, request, hooks).await) } } pub fn get_api_key_env_var( model: &str, custom_llm_provider: Option<&str>, -) -> Result, super::Error> { +) -> Result, Error> { Ok(resolve_provider_config(model, custom_llm_provider)? .1 .get_api_key_env_var()) @@ -134,7 +132,7 @@ pub fn get_api_key_env_var( pub fn get_health_check_document( model: &str, custom_llm_provider: Option<&str>, -) -> Result { +) -> Result { Ok(resolve_provider_config(model, custom_llm_provider)? .1 .get_health_check_document()) @@ -153,7 +151,7 @@ pub(crate) enum OcrProvider { pub(crate) fn resolve_provider_config( model: &str, custom_llm_provider: Option<&str>, -) -> Result<(String, OcrConfigKind), super::Error> { +) -> Result<(String, OcrConfigKind), Error> { let provider = get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider { model, @@ -162,7 +160,7 @@ pub(crate) fn resolve_provider_config( let ocr_provider = provider .custom_llm_provider .parse::() - .map_err(|_| super::Error::InvalidProvider(provider.custom_llm_provider.to_string()))?; + .map_err(|_| Error::InvalidProvider(provider.custom_llm_provider.to_string()))?; let config = match ocr_provider { OcrProvider::Cohere => OcrConfigKind::Cohere, OcrProvider::Mistral => OcrConfigKind::Mistral, @@ -196,6 +194,9 @@ fn is_document_intelligence_model(model: &str) -> bool { #[cfg(test)] mod tests { use litellm_auth::{InputSource, Sourced}; + use litellm_llms::{ + base_llm::ocr::document::InlineDocument, cohere::ocr::transformation::validate_document, + }; use rstest::rstest; use super::*; @@ -218,7 +219,7 @@ mod tests { fn invalid_provider_names_are_rejected(#[case] provider: &str) { assert!(matches!( resolve_provider_config("model", Some(provider)), - Err(crate::ocr::Error::InvalidProvider(value)) if value == provider + Err(Error::InvalidProvider(value)) if value == provider )); } @@ -232,9 +233,7 @@ mod tests { fn pdf_health_check_documents_are_valid(#[case] model: &str) { let document = get_health_check_document(model, None).unwrap(); assert!(matches!(document, OcrDocument::DocumentUrl { .. })); - let inline = crate::ocr::document::InlineDocument::parse(document.source()) - .unwrap() - .unwrap(); + let inline = InlineDocument::parse(document.source()).unwrap().unwrap(); assert_eq!(inline.mime_type().to_string(), "application/pdf"); assert!(inline.decode(4096).unwrap().starts_with(b"%PDF-")); } @@ -244,10 +243,8 @@ mod tests { #[case("azure_ai/cohere-parse")] fn png_health_check_documents_are_valid(#[case] model: &str) { let document = get_health_check_document(model, None).unwrap(); - crate::llms::cohere::ocr::validate_document(&document).unwrap(); - let inline = crate::ocr::document::InlineDocument::parse(document.source()) - .unwrap() - .unwrap(); + validate_document(&document).unwrap(); + let inline = InlineDocument::parse(document.source()).unwrap().unwrap(); assert_eq!(inline.mime_type().to_string(), "image/png"); assert!( inline @@ -435,9 +432,7 @@ mod tests { #[case] provider: Option<&str>, ) { let error = resolve_provider_config(model, provider).unwrap_err(); - assert!( - matches!(&error, crate::ocr::Error::InvalidProvider(provider) if provider == "not_a_provider") - ); + assert!(matches!(&error, Error::InvalidProvider(provider) if provider == "not_a_provider")); assert_eq!(error.http_status_code(), Some(400)); } } diff --git a/litellm-rust/crates/core/src/ocr/route.rs b/litellm-rust/crates/core/src/ocr/route.rs index 50058ac90fa..ac4237651da 100644 --- a/litellm-rust/crates/core/src/ocr/route.rs +++ b/litellm-rust/crates/core/src/ocr/route.rs @@ -5,13 +5,16 @@ use litellm_callbacks::{ event::{CallEvent, RequestContext, WireRequest}, route::Route, }; - -use super::{ - Error, LiteLLMOcrRequest, LiteLLMOcrResponse, OcrClient, - handler::perform_ocr_request, - types::{OcrDocumentInput, OcrFileContent, ResolvedOcrRequest}, +use litellm_llms::{ + base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse}, + custom_httpx::llm_http_handler::OcrClient, +}; + +use super::handler::perform_ocr_request; +use crate::{ + machine::{HostChannel, HostTokenProvider, MachineFault, RouteMachine, TokenRoute}, + ocr::types::{LiteLLMOcrRequest, OcrDocumentInput, OcrFileContent, ResolvedOcrRequest}, }; -use crate::machine::{HostChannel, HostTokenProvider, MachineFault, RouteMachine, TokenRoute}; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum OcrOp { diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs index 91851540c26..75202ed52a5 100644 --- a/litellm-rust/crates/core/src/ocr/types.rs +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -1,69 +1,17 @@ use std::{collections::BTreeMap, path::PathBuf, time::Duration}; use bytes::Bytes; -use litellm_auth::{InputSource, Sourced, TokenProviderHandle}; -use serde::{Deserialize, Serialize}; +use litellm_auth::{InputSource, TokenProviderHandle}; +use litellm_core_utils::call_arguments::CallArguments; +use litellm_llms::base_llm::ocr::{ + error::Error, + transformation::{ + OcrCredentialInputs, OcrDocument, OcrResponseFormat, OcrTransportConfig, response_format, + }, +}; use serde_json::{Map, Value}; -use serde_with::serde_as; use super::provider_config::{OcrConfigKind, resolve_provider_config}; -use crate::{ - call_arguments::CallArguments, - constants::OCR_HTTP_TIMEOUT_SECS, - serde_compat::{FiniteF64, LaxI64}, -}; - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "type")] -pub enum OcrDocument { - #[serde(rename = "document_url")] - DocumentUrl { - document_url: String, - #[serde(flatten)] - extra_fields: BTreeMap>, - }, - #[serde(rename = "image_url")] - ImageUrl { - image_url: String, - #[serde(flatten)] - extra_fields: BTreeMap>, - }, -} - -impl OcrDocument { - pub(crate) fn source(&self) -> &str { - match self { - Self::DocumentUrl { document_url, .. } => document_url, - Self::ImageUrl { image_url, .. } => image_url, - } - } - - pub(crate) fn is_remote(&self) -> bool { - let source = self.source(); - source.starts_with("http://") || source.starts_with("https://") - } - - pub(crate) fn with_source(self, source: String) -> Self { - match self { - Self::DocumentUrl { extra_fields, .. } => Self::DocumentUrl { - document_url: source, - extra_fields, - }, - Self::ImageUrl { extra_fields, .. } => Self::ImageUrl { - image_url: source, - extra_fields, - }, - } - } -} - -impl TryFrom for OcrDocument { - type Error = super::Error; - - fn try_from(value: Value) -> Result { - super::json::decode_request_value(value, "document") - } -} #[derive(Clone, Debug, PartialEq)] pub enum OcrDocumentInput { @@ -103,83 +51,6 @@ pub struct OcrFileContent { pub file_name: Option, } -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum OcrResponseFormat { - #[default] - Litellm, - Native, -} - -#[derive(Clone, Default)] -pub struct OcrCredentialInputs { - pub api_key: Option>, - pub dynamic_api_key: Option>, - pub api_base: Option>, - pub dynamic_api_base: Option>, -} - -impl OcrCredentialInputs { - pub fn new( - api_key: Option, - api_key_source: InputSource, - api_base: Option, - api_base_source: InputSource, - ) -> Self { - Self { - api_key: nonblank(api_key).map(|value| Sourced::new(value, api_key_source)), - dynamic_api_key: None, - api_base: nonblank(api_base).map(|value| Sourced::new(value, api_base_source)), - dynamic_api_base: None, - } - } -} - -#[derive(Clone)] -pub struct OcrTransportConfig { - pub extra_headers: Vec<(String, String)>, - pub extra_headers_source: InputSource, - pub timeout: Duration, - pub max_download_bytes: u64, - pub max_response_bytes: usize, - pub poll_timeout: Duration, -} - -impl Default for OcrTransportConfig { - fn default() -> Self { - Self { - extra_headers: Vec::new(), - extra_headers_source: InputSource::Deployment, - timeout: Duration::from_secs(OCR_HTTP_TIMEOUT_SECS), - max_download_bytes: crate::constants::OCR_DOWNLOAD_MAX_BYTES, - max_response_bytes: crate::constants::OCR_RESPONSE_MAX_BYTES, - poll_timeout: Duration::from_secs(crate::constants::OCR_POLL_TIMEOUT_SECS), - } - } -} - -impl OcrTransportConfig { - pub fn with_overrides( - self, - extra_headers: Vec<(String, String)>, - extra_headers_source: InputSource, - timeout: Option, - ) -> Self { - Self { - extra_headers, - extra_headers_source, - timeout: timeout.unwrap_or(self.timeout), - ..self - } - } -} - -fn nonblank(value: Option) -> Option { - value - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) -} - /// Caller-supplied connection overrides for a [`LiteLLMOcrRequest`], in the /// shape hosts receive them: JSON-ish headers, optional timeout, optional /// credentials, and per-field provenance in `input_sources`. @@ -197,14 +68,14 @@ impl OcrConnectionInputs { self.input_sources.get(name).copied().unwrap_or_default() } - fn header_pairs(&self) -> Result, super::Error> { + fn header_pairs(&self) -> Result, Error> { self.extra_headers .iter() .map(|(name, value)| { value .as_str() .map(|value| (name.clone(), value.to_string())) - .ok_or_else(|| super::Error::RequestField { + .ok_or_else(|| Error::RequestField { path: format!("extra_headers.{name}"), }) }) @@ -212,62 +83,6 @@ impl OcrConnectionInputs { } } -#[derive(Clone)] -pub struct OcrConnection { - pub api_key: Option, - pub api_key_source: InputSource, - pub api_base: Option, - pub api_base_source: InputSource, - pub extra_headers: Vec<(String, String)>, - pub extra_headers_source: InputSource, - pub timeout: Duration, - pub max_download_bytes: u64, - pub max_response_bytes: usize, - pub poll_timeout: Duration, -} - -impl OcrConnection { - pub(crate) fn new(credentials: ResolvedOcrCredentials, transport: OcrTransportConfig) -> Self { - let api_key_source = credentials - .api_key - .as_ref() - .map(Sourced::source) - .unwrap_or(InputSource::Deployment); - let api_base_source = credentials - .api_base - .as_ref() - .map(Sourced::source) - .unwrap_or(InputSource::Deployment); - Self { - api_key: credentials.api_key.map(Sourced::into_value), - api_key_source, - api_base: credentials.api_base.map(Sourced::into_value), - api_base_source, - extra_headers: transport.extra_headers, - extra_headers_source: transport.extra_headers_source, - timeout: transport.timeout, - max_download_bytes: transport.max_download_bytes, - max_response_bytes: transport.max_response_bytes, - poll_timeout: transport.poll_timeout, - } - } -} - -impl Default for OcrConnection { - fn default() -> Self { - Self::new( - ResolvedOcrCredentials::default(), - OcrTransportConfig::default(), - ) - } -} - -#[derive(Clone, Default)] -pub(crate) struct ResolvedOcrCredentials { - pub api_key: Option>, - pub api_base: Option>, -} - pub struct LiteLLMOcrRequest { pub model: String, pub document: D, @@ -285,7 +100,7 @@ impl LiteLLMOcrRequest { document: impl Into, custom_llm_provider: Option<&str>, optional_params: CallArguments, - ) -> Result { + ) -> Result { let (model, config) = resolve_provider_config(&model, custom_llm_provider)?; let default_transport = OcrTransportConfig::default(); let max_response_bytes = optional_params @@ -295,7 +110,7 @@ impl LiteLLMOcrRequest { .as_u64() .and_then(|value| usize::try_from(value).ok()) .filter(|value| *value > 0 && *value <= default_transport.max_response_bytes) - .ok_or_else(|| super::Error::RequestField { + .ok_or_else(|| Error::RequestField { path: "max_response_bytes".into(), }) }) @@ -353,15 +168,8 @@ impl LiteLLMOcrRequest { } } - pub(crate) fn response_format(&self) -> Result { - self.optional_params - .get("req_format") - .filter(|value| !value.is_null()) - .map(|value| { - serde_json::from_value(value.clone()).map_err(|_| super::Error::RequestFormat) - }) - .transpose() - .map(|format| format.unwrap_or_default()) + pub(crate) fn response_format(&self) -> Result { + response_format(&self.optional_params) } pub fn provider_name(&self) -> &'static str { @@ -395,7 +203,7 @@ impl LiteLLMOcrRequest { custom_llm_provider: Option<&str>, optional_params: CallArguments, connection: OcrConnectionInputs, - ) -> Result { + ) -> Result { let request = Self::new(model, document, custom_llm_provider, optional_params)?; let transport = request.transport.clone().with_overrides( connection.header_pairs()?, @@ -416,155 +224,6 @@ impl LiteLLMOcrRequest { pub(crate) type ResolvedOcrRequest = LiteLLMOcrRequest; -pub(crate) struct PreparedOcrRequest { - pub model: String, - pub document: OcrDocument, - pub connection: OcrConnection, - pub host: super::route::OcrHost, - /// Whether the caller handed over the document as is, so the wire body's document - /// is the caller's own input rather than something the route prepared. - pub caller_document: bool, - pub optional_params: CallArguments, - pub input_sources: BTreeMap, - pub azure_ad_token_provider: Option, - pub(crate) config: OcrConfigKind, -} - -impl PreparedOcrRequest { - pub(crate) fn new( - request: ResolvedOcrRequest, - connection: OcrConnection, - host: super::route::OcrHost, - caller_document: bool, - ) -> Self { - let LiteLLMOcrRequest { - model, - document, - credentials: _, - transport: _, - optional_params, - input_sources, - azure_ad_token_provider, - config, - } = request; - Self { - model, - document, - connection, - host, - caller_document, - optional_params, - input_sources, - azure_ad_token_provider, - config, - } - } - - pub(crate) fn response_format(&self) -> Result { - self.optional_params - .get("req_format") - .filter(|value| !value.is_null()) - .map(|value| { - serde_json::from_value(value.clone()).map_err(|_| super::Error::RequestFormat) - }) - .transpose() - .map(|format| format.unwrap_or_default()) - } - - pub(crate) fn provider_name(&self) -> &'static str { - self.config.provider().into() - } -} - -#[serde_as] -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct OcrPageDimensions { - #[serde_as(deserialize_as = "Option")] - pub dpi: Option, - #[serde_as(deserialize_as = "Option")] - pub height: Option, - #[serde_as(deserialize_as = "Option")] - pub width: Option, -} - -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct OcrPageImage { - pub image_base64: Option, - pub bbox: Option>, - #[serde(flatten)] - pub extra_fields: Map, -} - -#[serde_as] -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct OcrPage { - #[serde_as(deserialize_as = "LaxI64")] - pub index: i64, - pub markdown: String, - pub images: Option>, - pub dimensions: Option, - #[serde(flatten)] - pub extra_fields: Map, -} - -#[serde_as] -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct OcrUsageInfo { - #[serde_as(deserialize_as = "Option")] - pub pages_processed: Option, - #[serde_as(deserialize_as = "Option")] - pub pages_processed_annotation: Option, - #[serde_as(deserialize_as = "Option")] - pub credits: Option, - #[serde_as(deserialize_as = "Option")] - pub doc_size_bytes: Option, - #[serde(flatten)] - pub extra_fields: Map, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct LiteLLMOcrResponse { - pub pages: Vec, - pub model: String, - pub document_annotation: Option, - pub usage_info: Option, - pub content: Option, - pub tables: Option>>, - #[serde(rename = "keyValuePairs")] - pub key_value_pairs: Option>>, - #[serde(default = "ocr_object")] - pub object: String, - #[serde(flatten)] - pub extra_fields: Map, - #[serde(skip_serializing_if = "Option::is_none")] - pub provider_native_response: Option>, -} - -impl LiteLLMOcrResponse { - pub fn new(model: impl Into, pages: Vec) -> Self { - Self { - pages, - model: model.into(), - document_annotation: None, - usage_info: None, - content: None, - tables: None, - key_value_pairs: None, - object: ocr_object(), - extra_fields: Map::new(), - provider_native_response: None, - } - } - - pub fn into_json(self) -> Value { - serde_json::to_value(self).expect("OCR response fields are JSON-compatible") - } -} - -fn ocr_object() -> String { - "ocr".into() -} - #[cfg(test)] mod tests { use serde_json::json; @@ -645,97 +304,7 @@ mod tests { }; assert!(matches!( error, - super::super::Error::RequestField { ref path } if path == "extra_headers.x-a" + Error::RequestField { ref path } if path == "extra_headers.x-a" )); } - - #[test] - fn normalized_response_rejects_invalid_shared_fields() { - for fields in [ - json!({"pages":[{}]}), - json!({"pages":[{"index":0,"markdown":false}]}), - json!({"pages":[{"index":0,"markdown":"","images":[{"bbox":[]}]}]}), - json!({"usage_info":{"pages_processed":1.5}}), - json!({"tables":[false]}), - json!({"keyValuePairs":[[]]}), - json!({"provider_native_response":[]}), - ] { - let payload: Map = json!({"model":"model", "pages":[]}) - .as_object() - .unwrap() - .iter() - .chain(fields.as_object().unwrap()) - .map(|(key, value)| (key.clone(), value.clone())) - .collect(); - assert!(serde_json::from_value::(Value::Object(payload)).is_err()); - } - assert!( - serde_json::from_value::(json!({ - "type":"image_url", "image_url":"https://example.com/image", "detail":42 - })) - .is_err() - ); - } - - #[test] - fn numeric_coercion_preserves_integer_precision_and_rejects_fractional_values() { - for (value, expected) in [ - (json!("9007199254740993.0"), 9_007_199_254_740_993), - (json!("+2.000"), 2), - (json!("1_000"), 1000), - (json!(true), 1), - (json!(2.0), 2), - ] { - let page: OcrPage = - serde_json::from_value(json!({"index":value,"markdown":""})).unwrap(); - assert_eq!(page.index, expected); - } - for value in [ - json!("1e2"), - json!(".0"), - json!("2."), - json!("_2"), - json!("2__0"), - json!(2.5), - json!(null), - ] { - assert!( - serde_json::from_value::(json!({"index":value,"markdown":""})).is_err() - ); - } - } - - #[rstest::rstest] - #[case::document_url("document_url", "document_name", "application/pdf")] - #[case::image_url("image_url", "detail", "image/png")] - fn document_variants_preserve_provider_fields_when_rewriting_sources( - #[case] kind: &str, - #[case] field: &str, - #[case] mime_type: &str, - #[values(json!("kept"), Value::Null)] extra: Value, - ) { - let original = "https://example.com/input"; - let replacement = format!("data:{mime_type};base64,AA=="); - let document: OcrDocument = - serde_json::from_value(json!({"type": kind, kind: original, field: extra})).unwrap(); - assert_eq!(document.source(), original); - assert_eq!( - serde_json::to_value(document.with_source(replacement.clone())).unwrap(), - json!({"type": kind, kind: replacement, field: extra}) - ); - } - - #[test] - fn response_serialization_flattens_extra_fields_and_omits_absent_native_response() { - let response = LiteLLMOcrResponse { - extra_fields: json!({"provider_field":"kept"}) - .as_object() - .unwrap() - .clone(), - ..LiteLLMOcrResponse::new("model", vec![]) - }; - let serialized = response.into_json(); - assert_eq!(serialized["provider_field"], "kept"); - assert!(serialized.get("provider_native_response").is_none()); - } } diff --git a/litellm-rust/crates/core/src/ocr/wire.rs b/litellm-rust/crates/core/src/ocr/wire.rs index 603e455ace1..29345e38885 100644 --- a/litellm-rust/crates/core/src/ocr/wire.rs +++ b/litellm-rust/crates/core/src/ocr/wire.rs @@ -1,20 +1,23 @@ use std::{collections::BTreeMap, time::Duration}; use litellm_auth::InputSource; +use litellm_llms::base_llm::ocr::{ + error::Error, + transformation::{OcrDocument, decode_request_value}, +}; use serde::Deserialize; use serde_json::{Map, Value}; -pub use super::is_supported_request; -use super::{Error, LiteLLMOcrRequest, OcrConnectionInputs, OcrDocument, OcrDocumentInput}; +use crate::ocr::types::{LiteLLMOcrRequest, OcrConnectionInputs, OcrDocumentInput}; pub fn consumed_optional_params( model: &str, provider: Option<&str>, -) -> Result, Error> { - let specs = super::consumed_optional_params(model, provider)?; +) -> Result, Error> { + let specs = crate::ocr::arguments::consumed_optional_params(model, provider)?; Ok(consumed_optional_param_names(model, provider)? .into_iter() - .map(|name| crate::call_arguments::ArgumentSpec { + .map(|name| litellm_core_utils::call_arguments::ArgumentSpec { name, secret: specs.iter().any(|spec| spec.name == name && spec.secret), }) @@ -25,7 +28,7 @@ pub fn consumed_optional_param_names( model: &str, provider: Option<&str>, ) -> Result, Error> { - let names = super::consumed_optional_param_names(model, provider)?; + let names = crate::ocr::arguments::consumed_optional_param_names(model, provider)?; let (_, config) = super::provider_config::resolve_provider_config(model, provider)?; if config == super::provider_config::OcrConfigKind::VertexDeepSeek { return Ok(names @@ -99,7 +102,7 @@ pub fn decode_document(value: Value) -> Result { { return Err(Error::MissingDocumentUrl); } - super::json::decode_request_value(value, "document") + decode_request_value(value, "document") } #[cfg(test)] @@ -108,6 +111,7 @@ mod tests { use serde_json::json; use super::*; + use crate::ocr::arguments::is_supported_request; #[rstest] #[case::omitted(json!({"type":"document_url", "document_url":"https://example.com/a.pdf"}))] diff --git a/litellm-rust/crates/core/src/responses/error.rs b/litellm-rust/crates/core/src/responses/error.rs index 8bea035f0b0..677db2e08de 100644 --- a/litellm-rust/crates/core/src/responses/error.rs +++ b/litellm-rust/crates/core/src/responses/error.rs @@ -11,7 +11,7 @@ pub enum Error { #[error(transparent)] Auth(#[from] litellm_auth::Error), #[error(transparent)] - Transport(#[from] crate::transport::Error), + Transport(#[from] litellm_llms::custom_httpx::transport::Error), #[error(transparent)] - Headers(#[from] crate::http_utils::HeaderError), + Headers(#[from] litellm_llms::custom_httpx::http_handler::HeaderError), } diff --git a/litellm-rust/crates/core/src/responses/mod.rs b/litellm-rust/crates/core/src/responses/mod.rs index 6af2bf0c199..bc0f71896e5 100644 --- a/litellm-rust/crates/core/src/responses/mod.rs +++ b/litellm-rust/crates/core/src/responses/mod.rs @@ -1,4 +1,3 @@ mod error; pub use error::Error; -pub mod types; pub mod websocket; diff --git a/litellm-rust/crates/core/src/responses/websocket.rs b/litellm-rust/crates/core/src/responses/websocket.rs index 7758cb2414c..ccf4aa75149 100644 --- a/litellm-rust/crates/core/src/responses/websocket.rs +++ b/litellm-rust/crates/core/src/responses/websocket.rs @@ -6,6 +6,7 @@ use std::{ }; use futures_util::{SinkExt, StreamExt}; +use litellm_types::responses::streaming_websocket::ResponsesWsEventType; use rustls::{ClientConfig, RootCertStore}; use tokio::{net::TcpStream, sync::Mutex}; use tokio_tungstenite::{ @@ -20,122 +21,6 @@ use tokio_tungstenite::{ }; use super::Error; -use crate::{ - constants::{OPENAI_RESPONSES_DEFAULT_API_BASE, OPENAI_RESPONSES_PATH}, - responses::types::{ResponsesWsEvent, ResponsesWsEventType, ResponsesWsTransformResult}, -}; - -pub trait ResponsesWebSocketProviderConfig: Sync { - fn supports_native_websocket(&self) -> bool { - false - } - - fn model_in_websocket_url(&self) -> bool { - true - } - - fn complete_websocket_url(&self, api_base: Option<&str>, model: &str) -> String { - complete_websocket_url(api_base, model, self.model_in_websocket_url()) - } - - fn transform_ws_request( - &self, - event: &ResponsesWsEvent, - model: &str, - ) -> Result; - - fn transform_ws_response( - &self, - event: &ResponsesWsEvent, - model: &str, - ) -> Result; -} - -pub fn complete_websocket_url( - api_base: Option<&str>, - model: &str, - model_in_websocket_url: bool, -) -> String { - let base = api_base - .map(str::trim) - .filter(|value| !value.is_empty()) - .unwrap_or(OPENAI_RESPONSES_DEFAULT_API_BASE); - let (base_without_query, query) = base - .split_once('?') - .map_or((base, None), |(value, query)| (value, Some(query))); - let response_url = format!( - "{}{}", - base_without_query.trim_end_matches('/'), - OPENAI_RESPONSES_PATH - ); - let scheme_flipped = if let Some(rest) = response_url.strip_prefix("https://") { - format!("wss://{rest}") - } else if let Some(rest) = response_url.strip_prefix("http://") { - format!("ws://{rest}") - } else { - response_url - }; - let url = query.map_or(scheme_flipped.clone(), |value| { - format!("{scheme_flipped}?{value}") - }); - if !model_in_websocket_url - || query.is_some_and(|value| { - value - .split('&') - .any(|part| part.split('=').next() == Some("model")) - }) - { - return url; - } - format!( - "{url}{}model={}", - if query.is_some() { "&" } else { "?" }, - percent_encode(model) - ) -} - -fn percent_encode(value: &str) -> String { - value - .bytes() - .map(|byte| { - if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { - format!("{}", byte as char) - } else { - format!("%{byte:02X}") - } - }) - .collect() -} - -pub fn enforce_model(event: &ResponsesWsEvent, model: &str) -> ResponsesWsEvent { - if !event.is_response_create() { - return event.clone(); - } - let mut enforced = event.clone(); - let has_flat_model = enforced.data.contains_key("model"); - if let Some(response) = enforced - .data - .get_mut("response") - .and_then(serde_json::Value::as_object_mut) - { - response.insert( - "model".to_string(), - serde_json::Value::String(model.to_string()), - ); - if has_flat_model { - enforced.data.insert( - "model".to_string(), - serde_json::Value::String(model.to_string()), - ); - } - } else { - enforced.data.insert( - "model".to_string(), - serde_json::Value::String(model.to_string()), - ); - } - enforced -} pub fn is_terminal_event(event_type: &ResponsesWsEventType) -> bool { matches!( @@ -210,7 +95,9 @@ impl ResponsesWebSocketConnection { timeout: Option, ) -> Result { let mut request = url.into_client_request().map_err(|error| { - Error::Transport(crate::transport::Error::Network(error.to_string())) + Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( + error.to_string(), + )) })?; for (name, value) in headers { let header_name = name @@ -223,7 +110,7 @@ impl ResponsesWebSocketConnection { let connect = connect_upstream(request); let result = match timeout { Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| { - Error::Transport(crate::transport::Error::Network( + Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( "Responses WebSocket connection timed out".into(), )) })?, @@ -231,12 +118,14 @@ impl ResponsesWebSocketConnection { }; let (socket, _) = result.map_err(|error| match *error { tokio_tungstenite::tungstenite::Error::Http(response) => { - Error::Transport(crate::transport::Error::Http { + Error::Transport(litellm_llms::custom_httpx::transport::Error::Http { status: response.status().as_u16(), body: String::new(), }) } - other => Error::Transport(crate::transport::Error::Network(other.to_string())), + other => Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( + other.to_string(), + )), })?; Ok(Self { socket: Arc::new(Mutex::new(Some(socket))), @@ -246,14 +135,17 @@ impl ResponsesWebSocketConnection { pub async fn send_text(&self, text: String) -> Result<(), Error> { let mut socket = self.socket.lock().await; let Some(socket) = socket.as_mut() else { - return Err(Error::Transport(crate::transport::Error::Network( - "Responses WebSocket is closed".into(), - ))); + return Err(Error::Transport( + litellm_llms::custom_httpx::transport::Error::Network( + "Responses WebSocket is closed".into(), + ), + )); }; - socket - .send(Message::Text(text)) - .await - .map_err(|error| Error::Transport(crate::transport::Error::Network(error.to_string()))) + socket.send(Message::Text(text)).await.map_err(|error| { + Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( + error.to_string(), + )) + }) } pub async fn recv_text(&self) -> Result, Error> { @@ -268,9 +160,9 @@ impl ResponsesWebSocketConnection { .map_err(|error| Error::InvalidResponse(error.to_string())), Some(Ok(Message::Close(_))) | None => Ok(None), Some(Ok(_)) => Ok(None), - Some(Err(error)) => Err(Error::Transport(crate::transport::Error::Network( - error.to_string(), - ))), + Some(Err(error)) => Err(Error::Transport( + litellm_llms::custom_httpx::transport::Error::Network(error.to_string()), + )), } } @@ -278,72 +170,12 @@ impl ResponsesWebSocketConnection { let mut socket = self.socket.lock().await; if let Some(socket) = socket.as_mut() { socket.close(None).await.map_err(|error| { - Error::Transport(crate::transport::Error::Network(error.to_string())) + Error::Transport(litellm_llms::custom_httpx::transport::Error::Network( + error.to_string(), + )) })?; } *socket = None; Ok(()) } } - -#[cfg(test)] -mod tests { - use super::*; - - fn event(value: serde_json::Value) -> ResponsesWsEvent { - serde_json::from_value(value).expect("valid event") - } - - #[test] - fn url_construction_matches_python_defaults_and_query_behavior() { - assert_eq!( - complete_websocket_url(None, "gpt-5", true), - "wss://api.openai.com/v1/responses?model=gpt-5" - ); - assert_eq!( - complete_websocket_url(Some("http://localhost:8080/"), "gpt 5", true), - "ws://localhost:8080/responses?model=gpt%205" - ); - assert_eq!( - complete_websocket_url(Some("https://example.test/v1?foo=bar"), "gpt-5", true), - "wss://example.test/v1/responses?foo=bar&model=gpt-5" - ); - assert_eq!( - complete_websocket_url(Some("https://example.test?model=existing"), "gpt-5", true), - "wss://example.test/responses?model=existing" - ); - } - - #[test] - fn enforce_model_overrides_flat_and_nested_values() { - let flat = enforce_model( - &event(serde_json::json!({"type":"response.create","model":"wrong"})), - "gpt-5", - ); - assert_eq!(flat.model(), Some("gpt-5")); - let nested = enforce_model( - &event(serde_json::json!({ - "type":"response.create", - "model":"wrong", - "response":{"model":"also-wrong"} - })), - "gpt-5", - ); - assert_eq!(nested.model(), Some("gpt-5")); - assert_eq!( - nested - .data - .get("response") - .and_then(|value| value.get("model")), - Some(&serde_json::json!("gpt-5")) - ); - let nested_without_flat = enforce_model( - &event(serde_json::json!({ - "type":"response.create", - "response":{"model":"also-wrong"} - })), - "gpt-5", - ); - assert!(!nested_without_flat.data.contains_key("model")); - } -} diff --git a/litellm-rust/crates/core/src/transport/mod.rs b/litellm-rust/crates/core/src/transport/mod.rs deleted file mode 100644 index 0405e9de3c3..00000000000 --- a/litellm-rust/crates/core/src/transport/mod.rs +++ /dev/null @@ -1,2 +0,0 @@ -mod error; -pub use error::Error; diff --git a/litellm-rust/crates/core/tests/azure_ai_ocr.rs b/litellm-rust/crates/core/tests/azure_ai_ocr.rs index ad46abc9ccd..1492aaaeb11 100644 --- a/litellm-rust/crates/core/tests/azure_ai_ocr.rs +++ b/litellm-rust/crates/core/tests/azure_ai_ocr.rs @@ -1,9 +1,8 @@ +use litellm_llms::base_llm::ocr::error::Error; use serde_json::{Value, json}; -use super::{ - LocalOcrHost, - test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request}, -}; +use super::test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request}; +use crate::ocr::route::LocalOcrHost; #[tokio::test] async fn facade_executes_azure_mistral_with_prepared_auth() { @@ -80,3 +79,215 @@ async fn rejects_non_inline_body_after_guardrails() { let error = perform_ocr_with(host).await.unwrap_err(); assert!(error.to_string().contains("data URI")); } + +mod transformation { + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + + use litellm_auth::{ + ResolvedCredential, SecretValue, TokenFuture, TokenProvider, TokenProviderHandle, + }; + use rstest::rstest; + use serde_json::json; + + use super::*; + use crate::ocr::{ + test_support::{MockResponse, header, mock_server, perform_ocr}, + types::LiteLLMOcrRequest, + wire::decode_request, + }; + + #[derive(Debug)] + struct CountingToken { + token: fn(usize) -> String, + calls: AtomicUsize, + } + + impl CountingToken { + fn new(token: fn(usize) -> String) -> Arc { + Arc::new(Self { + token, + calls: AtomicUsize::new(0), + }) + } + + fn calls(&self) -> usize { + self.calls.load(Ordering::SeqCst) + } + } + + impl TokenProvider for CountingToken { + fn acquire(&self) -> TokenFuture<'_> { + let call = self.calls.fetch_add(1, Ordering::SeqCst) + 1; + let token = SecretValue::new((self.token)(call)); + Box::pin(async move { + Ok(ResolvedCredential::AccessToken { + token, + expires_on: None, + }) + }) + } + } + + fn numbered_token(call: usize) -> String { + format!("callback-{call}") + } + + fn azure_request( + provider: &Arc, + api_base: Option<&str>, + api_key: Option<&str>, + extra_headers: Value, + optional_params: Value, + ) -> LiteLLMOcrRequest { + let wire = serde_json::from_value(json!({ + "model": "azure_ai/mistral-ocr-latest", + "document": {"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}, + "api_key": api_key, + "api_base": api_base, + "custom_llm_provider": null, + "extra_headers": extra_headers, + "optional_params": optional_params, + "timeout_seconds": 2.0 + })) + .unwrap(); + LiteLLMOcrRequest { + azure_ad_token_provider: Some(TokenProviderHandle::new(provider.clone())), + ..decode_request(wire).unwrap() + } + } + + fn ocr_page() -> MockResponse { + MockResponse::json(json!({"pages":[{"index":0,"markdown":"hello"}]})) + } + + #[tokio::test] + async fn token_provider_result_is_the_bearer_and_is_acquired_for_each_request() { + let provider = CountingToken::new(numbered_token); + let (base, seen, server) = mock_server(vec![ocr_page(), ocr_page()]).await; + + for _ in 0..2 { + perform_ocr(azure_request( + &provider, + Some(&base), + None, + Value::Null, + json!({}), + )) + .await + .unwrap(); + } + server.await.unwrap(); + + assert_eq!(provider.calls(), 2); + let requests = seen.lock().unwrap(); + assert_eq!( + requests + .iter() + .map(|request| header(request, "authorization")) + .collect::>(), + [Some("Bearer callback-1"), Some("Bearer callback-2")] + ); + } + + #[rstest] + #[case::api_key_skips_provider(Some("resource-key"), Value::Null, json!({}), "Bearer resource-key", 0)] + #[case::provider_beats_static_token( + None, + Value::Null, + json!({"azure_ad_token":"static-token"}), + "Bearer callback-1", + 1 + )] + #[case::header_wins_on_the_wire_but_provider_still_runs( + None, + json!({"Authorization":"Bearer override"}), + json!({}), + "Bearer override", + 1 + )] + #[tokio::test] + async fn credential_precedence( + #[case] api_key: Option<&str>, + #[case] extra_headers: Value, + #[case] optional_params: Value, + #[case] expected_authorization: &str, + #[case] expected_calls: usize, + ) { + let provider = CountingToken::new(numbered_token); + let (base, seen, server) = mock_server(vec![ocr_page()]).await; + + perform_ocr(azure_request( + &provider, + Some(&base), + api_key, + extra_headers, + optional_params, + )) + .await + .unwrap(); + server.await.unwrap(); + + assert_eq!(provider.calls(), expected_calls); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert_eq!( + header(&requests[0], "authorization"), + Some(expected_authorization) + ); + } + + #[rstest] + #[case::missing_api_base( + false, + json!({}), + numbered_token, + |error: &Error| matches!(error, Error::Auth(litellm_auth::Error::MissingApiBase { + provider: "Azure AI", + environment_variable: "AZURE_AI_API_BASE", + })), + 0 + )] + #[case::unsupported_oidc_reference( + true, + json!({"azure_ad_token":"oidc/assertion","client_id":"client","tenant_id":"tenant"}), + numbered_token, + |error: &Error| matches!(error, Error::Auth(litellm_auth::Error::UnsupportedOidcReference)), + 0 + )] + #[case::empty_provider_token_ignores_static_token( + true, + json!({"azure_ad_token":"static-token"}), + |_| String::new(), + |error: &Error| matches!(error, Error::MissingAzureAiCredentials), + 1 + )] + #[tokio::test] + async fn credential_failures_send_no_provider_request( + #[case] with_api_base: bool, + #[case] optional_params: Value, + #[case] token: fn(usize) -> String, + #[case] expected: fn(&Error) -> bool, + #[case] expected_calls: usize, + ) { + let provider = CountingToken::new(token); + let (base, seen, server) = mock_server(vec![ocr_page()]).await; + + let error = perform_ocr(azure_request( + &provider, + with_api_base.then_some(base.as_str()), + None, + Value::Null, + optional_params, + )) + .await + .unwrap_err(); + server.abort(); + + assert!(expected(&error), "unexpected error: {error:?}"); + assert_eq!(provider.calls(), expected_calls); + assert!(seen.lock().unwrap().is_empty()); + } +} diff --git a/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs b/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs index 6039ee2bfe4..01a4e5efb3b 100644 --- a/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs +++ b/litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs @@ -1,12 +1,13 @@ use litellm_callbacks::event::CallEvent; +use litellm_llms::base_llm::ocr::error::Error; use rstest::rstest; use serde_json::{Value, json}; use super::{ - LocalOcrHost, test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request}, wire::{OcrWireRequest, decode_request}, }; +use crate::ocr::route::LocalOcrHost; fn query_value(url: &str, key: &str) -> Option { url::Url::parse(url) @@ -27,12 +28,13 @@ async fn facade_maps_pages_features_and_url_document() { &base, json!({"pages":[2,0,0,1],"features":["keyValuePairs","languages"]}), ); - request.document = serde_json::from_value::(json!({ - "type":"document_url", - "document_url":"https://example.com/document.pdf" - })) - .unwrap() - .into(); + request.document = + serde_json::from_value::(json!({ + "type":"document_url", + "document_url":"https://example.com/document.pdf" + })) + .unwrap() + .into(); perform_ocr(request).await.unwrap(); server.await.unwrap(); @@ -52,16 +54,16 @@ async fn facade_maps_pages_features_and_url_document() { } #[rstest] -#[case(json!({"pages":[true]}), crate::ocr::Error::Pages("expected only integers or only strings".into()))] -#[case(json!({"pages":[1,"2"]}), crate::ocr::Error::Pages("expected only integers or only strings".into()))] -#[case(json!({"pages":[-1]}), crate::ocr::Error::Pages("negative page index".into()))] -#[case(json!({"pages":"1&&features=bad"}), crate::ocr::Error::Pages("invalid native page range".into()))] -#[case(json!({"features":"languages&pages=1"}), crate::ocr::Error::Features)] -#[case(json!({"req_format":"azure"}), crate::ocr::Error::RequestFormat)] +#[case(json!({"pages":[true]}), Error::Pages("expected only integers or only strings".into()))] +#[case(json!({"pages":[1,"2"]}), Error::Pages("expected only integers or only strings".into()))] +#[case(json!({"pages":[-1]}), Error::Pages("negative page index".into()))] +#[case(json!({"pages":"1&&features=bad"}), Error::Pages("invalid native page range".into()))] +#[case(json!({"features":"languages&pages=1"}), Error::Features)] +#[case(json!({"req_format":"azure"}), Error::RequestFormat)] #[tokio::test] async fn rejects_invalid_pages_features_and_format( #[case] options: Value, - #[case] expected: super::Error, + #[case] expected: Error, ) { let (base, seen, server) = mock_server(vec![MockResponse::json(json!({}))]).await; let result = decode_request(OcrWireRequest { @@ -460,3 +462,207 @@ async fn model_id_is_encoded_and_dot_segments_are_rejected() { assert!(error.to_string().contains("dot segment")); } } + +mod transformation { + use std::sync::{Arc, Mutex}; + + use litellm_callbacks::event::CallEvent; + use litellm_llms::base_llm::ocr::transformation::OcrDocument; + use serde_json::{Value, json}; + + use super::*; + use crate::ocr::{ + route::LocalOcrHost, + test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request}, + }; + + #[tokio::test] + async fn facade_maps_pages_features_and_url_document() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ + "status":"succeeded", + "analyzeResult":{"pages":[]} + }))]) + .await; + let mut request = wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({"pages":[2,0,0,1],"features":["keyValuePairs","languages"], "future_option": {"nested":null}, "extra_body":{"provider_option":false}}), + ); + request.document = serde_json::from_value::(json!({ + "type":"document_url", + "document_url":"https://example.com/document.pdf" + })) + .unwrap() + .into(); + + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + let request = &seen.lock().unwrap()[0]; + let target = request.split_whitespace().nth(1).unwrap(); + let url = format!("{base}{target}"); + assert_eq!(query_value(&url, "pages").as_deref(), Some("1,2,3")); + assert_eq!( + query_value(&url, "features").as_deref(), + Some("keyValuePairs,languages") + ); + let body: Value = serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap(); + assert_eq!( + body, + json!({"urlSource":"https://example.com/document.pdf", "future_option":{"nested":null}, "provider_option":false}) + ); + } + + #[tokio::test] + async fn rejects_invalid_pages_features_and_format() { + for options in [ + json!({"pages":[true]}), + json!({"pages":[1,"2"]}), + json!({"pages":[-1]}), + json!({"pages":"1&&features=bad"}), + json!({"features":"languages&pages=1"}), + json!({"req_format":"azure"}), + ] { + let request = wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + "http://127.0.0.1:1", + options.clone(), + ); + let rejected = perform_ocr(request).await.is_err(); + assert!(rejected, "accepted {options}"); + } + } + + #[tokio::test] + async fn immediate_response_normalizes_pages_and_preserves_native() { + let operation = json!({ + "status":"succeeded", + "operationExtension":42, + "analyzeResult":{ + "content":"A\n\nB", + "tables":[{"cells":[]}], + "keyValuePairs":[{"key":{"content":"A"}}], + "pages":[{ + "pageNumber":"2", + "width":"8.5", + "height":11, + "unit":"inch", + "lines":[{"content":"A"},{"content":null},{"content":"B"}] + }] + } + }); + let (base, _, server) = mock_server(vec![MockResponse::json(operation.clone())]).await; + let result = perform_ocr(wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({"req_format":"native"}), + )) + .await + .unwrap(); + server.await.unwrap(); + + assert_eq!(result.pages[0].index, 1); + assert_eq!(result.pages[0].markdown, "A\n\nB"); + assert_eq!( + serde_json::to_value(&result.pages[0].dimensions).unwrap(), + json!({"width":816,"height":1056,"dpi":96}) + ); + assert_eq!(result.usage_info.as_ref().unwrap().pages_processed, Some(1)); + let serialized = result.clone().into_json(); + assert_eq!(serialized["content"], "A\n\nB"); + assert_eq!(serialized["tables"], json!([{"cells":[]}])); + assert_eq!( + serialized["keyValuePairs"], + json!([{"key":{"content":"A"}}]) + ); + assert!(serialized.get("key_value_pairs").is_none()); + assert_eq!( + result.provider_native_response.as_ref(), + operation.as_object() + ); + } + + #[tokio::test] + async fn accepted_response_polls_to_success_with_only_credentials() { + let operation = json!({"status":"succeeded","analyzeResult":{"pages":[]}}); + let (base, seen, server) = mock_server(vec![ + MockResponse { + status: 202, + headers: vec![("Operation-Location", "{base}/operation".into())], + body: json!({}), + }, + MockResponse { + status: 200, + headers: vec![("Retry-After", "0".into())], + body: json!({"status":"running"}), + }, + MockResponse::json(operation.clone()), + ]) + .await; + let mut request = wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({"req_format":"native"}), + ); + request + .transport + .extra_headers + .push(("X-Trace".into(), "initial-only".into())); + + let result = perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!( + result.provider_native_response.as_ref(), + operation.as_object() + ); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 3); + assert!(requests[0].to_ascii_lowercase().contains("x-trace:")); + for poll in &requests[1..] { + assert!(!poll.to_ascii_lowercase().contains("x-trace:")); + assert!( + poll.to_ascii_lowercase() + .contains("ocp-apim-subscription-key: test-key") + ); + } + } + + #[tokio::test] + async fn accepted_response_emits_response_received_for_submission_and_completed_poll() { + let (base, seen, server) = mock_server(vec![ + MockResponse { + status: 202, + headers: vec![("Operation-Location", "{base}/operation".into())], + body: json!({"submitted": true}), + }, + MockResponse::json(json!({"status":"succeeded"})), + ]) + .await; + let responses_received = Arc::new(Mutex::new(Vec::new())); + let request_count = seen.clone(); + let observed = responses_received.clone(); + let host = LocalOcrHost::new(wire_request( + "azure_ai/doc-intelligence/prebuilt-read", + &base, + json!({}), + )) + .with_observer(move |event| { + if let CallEvent::ResponseReceived { raw } = event { + observed + .lock() + .unwrap() + .push((request_count.lock().unwrap().len(), raw.body.clone())); + } + }); + + perform_ocr_with(host).await.unwrap(); + server.await.unwrap(); + assert_eq!(seen.lock().unwrap().len(), 2); + assert_eq!( + *responses_received.lock().unwrap(), + [ + (1, r#"{"submitted":true}"#.to_string()), + (2, r#"{"status":"succeeded"}"#.to_string()), + ] + ); + } +} diff --git a/litellm-rust/crates/core/tests/cohere_ocr.rs b/litellm-rust/crates/core/tests/cohere_ocr.rs new file mode 100644 index 00000000000..fc1203f0980 --- /dev/null +++ b/litellm-rust/crates/core/tests/cohere_ocr.rs @@ -0,0 +1,136 @@ +mod transformation { + use litellm_llms::{ + base_llm::ocr::{ + error::Error, + transformation::{BaseOcrConfig, OcrDocument, OcrResponseFormat}, + }, + cohere::ocr::transformation::*, + }; + use rstest::rstest; + use serde_json::{Value, json}; + + #[tokio::test] + async fn composed_body_preserves_native_document_fields_and_untyped_overrides() { + let request = crate::ocr::test_support::wire_request( + "cohere/parse", + "https://example.com", + json!({ + "output_format":"markdown", "timeout":30, + "extra_body":{ + "output_format": {"future":true}, + "document":{"type":"image_url","image_url":"https://example.com/a.png", + "provider_options":{"nested":[false,0,null]}} + } + }), + ); + let request = request.with_document( + serde_json::from_value(json!({ + "type":"image_url","image_url":"https://example.com/original.png" + })) + .unwrap(), + ); + let request = crate::ocr::prepare::prepare_request_for_test(request); + let http = CohereParseConfig + .prepare_request( + &request, + &crate::ocr::test_support::ocr_client(), + &crate::ocr::test_support::NoHooks, + ) + .await + .unwrap(); + let body: Value = serde_json::from_slice(http.body().unwrap().as_bytes().unwrap()).unwrap(); + assert_eq!( + body, + json!({ + "model":"parse", "output_format":{"future":true}, + "document":{"type":"image_url","image_url":"https://example.com/a.png", + "provider_options":{"nested":[false,0,null]}} + }) + ); + } + + #[tokio::test] + async fn explicit_null_options_use_defaults_before_http() { + let request = crate::ocr::test_support::wire_request( + "cohere/parse", + "https://example.com", + json!({"output_format":null,"req_format":null}), + ); + let request = request.with_document( + serde_json::from_value( + json!({"type":"image_url","image_url":"https://example.com/a.png"}), + ) + .unwrap(), + ); + assert_eq!( + request.response_format().unwrap(), + OcrResponseFormat::Litellm + ); + let request = crate::ocr::prepare::prepare_request_for_test(request); + let http = CohereParseConfig + .prepare_request( + &request, + &crate::ocr::test_support::ocr_client(), + &crate::ocr::test_support::NoHooks, + ) + .await + .unwrap(); + let body: Value = serde_json::from_slice(http.body().unwrap().as_bytes().unwrap()).unwrap(); + assert_eq!(body["output_format"], "markdown"); + assert!(body.get("req_format").is_none()); + } + + #[rstest] + #[case::cohere("cohere/parse-v5.0", "POST /v2/parse ")] + #[case::azure_ai("azure_ai/Cohere-parse-v5.0", "POST /providers/cohere/v2/parse ")] + #[tokio::test] + async fn route_sends_image_to_its_parse_endpoint_with_the_bearer_key( + #[case] model: &str, + #[case] request_line: &str, + ) { + use crate::ocr::test_support::{MockResponse, header, mock_server, perform_ocr}; + + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; + let request = crate::ocr::test_support::wire_request(model, &base, json!({})) + .with_document( + serde_json::from_value::( + json!({"type":"image_url","image_url":"data:image/png;base64,YWJj"}), + ) + .unwrap() + .into(), + ); + + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with(request_line), "{}", requests[0]); + assert_eq!( + header(&requests[0], "authorization"), + Some("Bearer test-key") + ); + } + + #[rstest] + #[tokio::test] + async fn route_rejects_non_image_document_without_a_request( + #[values("cohere/parse-v5.0", "azure_ai/Cohere-parse-v5.0")] model: &str, + ) { + use crate::ocr::test_support::{MockResponse, mock_server, perform_ocr}; + + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; + + let error = perform_ocr(crate::ocr::test_support::wire_request( + model, + &base, + json!({}), + )) + .await + .unwrap_err(); + server.abort(); + + assert!(matches!(error, Error::CohereImageOnly), "{error:?}"); + assert!(seen.lock().unwrap().is_empty()); + } +} diff --git a/litellm-rust/crates/core/tests/deepseek_ocr.rs b/litellm-rust/crates/core/tests/deepseek_ocr.rs index 491978df75a..96e7451769d 100644 --- a/litellm-rust/crates/core/tests/deepseek_ocr.rs +++ b/litellm-rust/crates/core/tests/deepseek_ocr.rs @@ -1,17 +1,13 @@ +use litellm_llms::{ + base_llm::ocr::transformation::{BaseOcrConfig, OcrDocument}, + vertex_ai::ocr::deepseek_transformation::{ + DeepSeekOcrParams, DeepSeekOcrResponse, VertexAIDeepSeekOCRConfig, + normalize_response as transform_ocr_response, + }, +}; use rstest::rstest; use serde_json::{Value, json}; -use crate::{ - llms::{ - base_llm::ocr::transformation::BaseOcrConfig, - vertex_ai::ocr::deepseek_transformation::{ - DeepSeekOcrParams, DeepSeekOcrResponse, VertexAIDeepSeekOCRConfig, - normalize_response as transform_ocr_response, - }, - }, - ocr::types::OcrDocument, -}; - fn document() -> OcrDocument { serde_json::from_value(json!({"type":"image_url","image_url":"gs://bucket/a.png"})).unwrap() } diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index af88c5f6ec9..1f591d74d5d 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -5,16 +5,23 @@ use litellm_callbacks::{ host::{Host, HostOp, HostResult}, machine::{HostFailure, Machine, MachineStep}, }; +use litellm_llms::{ + base_llm::ocr::{ + error::Error as OcrError, + transformation::{LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig}, + }, + custom_httpx::llm_http_handler::OcrClient, +}; use rstest::rstest; use serde_json::{Value, json}; use super::{ - LocalOcrHost, OcrClient, OcrOp, OcrOpResult, ocr_machine, test_support::{ MockResponse, mock_server, ocr_client, perform_ocr, perform_ocr_with, wire_request, }, wire::{OcrWireRequest, decode_request}, }; +use crate::ocr::route::{LocalOcrHost, OcrOp, OcrOpResult, ocr_machine}; #[rstest] #[case::mistral("mistral/model", json!({}))] @@ -41,7 +48,7 @@ async fn ocr_contract_upstream_error_preserves_status_body_and_headers( .unwrap_err(); server.await.unwrap(); assert_eq!(seen.lock().unwrap().len(), 1); - let super::Error::Provider { + let OcrError::Provider { status, body, headers, @@ -175,11 +182,12 @@ async fn facade_uses_the_injected_http_client() { .default_headers(default_headers) .build() .unwrap(); - OcrClient::new(provider_http) - .unwrap() - .perform(wire_request("mistral/model", &base, json!({}))) - .await - .unwrap(); + crate::ocr::client::perform( + &OcrClient::new(provider_http).unwrap(), + wire_request("mistral/model", &base, json!({})), + ) + .await + .unwrap(); server.await.unwrap(); assert!(seen.lock().unwrap()[0].contains("x-transport-owner: host")); } @@ -193,7 +201,7 @@ fn event_name(event: &CallEvent) -> &'static str { } fn recording_host( - request: super::LiteLLMOcrRequest, + request: crate::ocr::types::LiteLLMOcrRequest, events: Arc>>, block: bool, ) -> LocalOcrHost { @@ -202,7 +210,7 @@ fn recording_host( .with_before_send(move |wire, _| { before_send_events.lock().unwrap().push("before_send"); if block { - return Err(crate::ocr::Error::InvalidRequest("blocked".into())); + return Err(OcrError::InvalidRequest("blocked".into())); } Ok(wire) }) @@ -259,7 +267,7 @@ async fn before_send_context_names_passthrough_fields_and_secrets() { &base, json!({"client_secret": "shh", "tenant_id": "t"}), ); - let request = request.with_document(super::OcrDocumentInput::Bytes { + let request = request.with_document(crate::ocr::types::OcrDocumentInput::Bytes { bytes: b"abc".as_slice().into(), file_name: None, mime_type: Some("application/pdf".into()), @@ -302,7 +310,7 @@ async fn lifecycle_blocking_prevents_execution_and_emits_one_failure() { true, ); let error = perform_ocr_with(host).await.unwrap_err(); - assert!(matches!(error, crate::ocr::Error::InvalidRequest(message) if message == "blocked")); + assert!(matches!(error, OcrError::InvalidRequest(message) if message == "blocked")); assert_eq!(*events.lock().unwrap(), ["before_send", "failure"]); } @@ -331,11 +339,11 @@ async fn upstream_failure_emits_one_terminal_failure() { async fn drive_until( client: OcrClient, host: &LocalOcrHost, - mut intercept: impl FnMut(WireRequest) -> Result>, + mut intercept: impl FnMut(WireRequest) -> Result>, ) -> ( - Result, + Result, Vec<&'static str>, - super::OcrMachine, + crate::ocr::route::OcrMachine, ) { let mut machine = ocr_machine(client); let mut result = None; @@ -386,13 +394,13 @@ async fn failed_before_send_does_not_replay_or_reach_transport() { json!({}), )); let (outcome, ops, mut machine) = drive_until(ocr_client(), &host, |_| { - Err(HostFailure::Error(crate::ocr::Error::InvalidRequest( + Err(HostFailure::Error(OcrError::InvalidRequest( "before_send failed".into(), ))) }) .await; assert!( - matches!(outcome, Err(crate::ocr::Error::InvalidRequest(message)) if message == "before_send failed") + matches!(outcome, Err(OcrError::InvalidRequest(message)) if message == "before_send failed") ); assert_eq!(ops, ["ProjectRequest", "BeforeSend"]); assert!(machine.resume(None).await.is_err()); @@ -413,7 +421,7 @@ async fn invalid_provider_response_emits_response_received_before_normalization_ ); let error = perform_ocr_with(host).await.unwrap_err(); server.await.unwrap(); - assert!(matches!(error, crate::ocr::Error::ResponseField { .. })); + assert!(matches!(error, OcrError::ResponseField { .. })); assert_eq!(seen.lock().unwrap().len(), 1); assert_eq!( *responses_received.lock().unwrap(), @@ -435,14 +443,14 @@ async fn direct_native_host_drives_the_same_state_machine() { assert_eq!(ops, ["ProjectRequest", "BeforeSend", "response"]); assert!(matches!( machine.resume(None).await, - Err(crate::ocr::Error::InvalidRequest(_)) + Err(OcrError::InvalidRequest(_)) )); } async fn drive_native_file_call( - request: super::LiteLLMOcrRequest, - content: Result, -) -> (Result, usize) { + request: crate::ocr::types::LiteLLMOcrRequest, + content: Result, +) -> (Result, usize) { let reads = Arc::new(Mutex::new(0)); let counted = reads.clone(); let content = Mutex::new(Some(content)); @@ -462,13 +470,13 @@ async fn host_reader_documents_are_read_once_at_the_core_selected_point_and_enco }))]) .await; let request = wire_request("mistral/model", &base, json!({})).with_document( - super::OcrDocumentInput::HostReader { + crate::ocr::types::OcrDocumentInput::HostReader { mime_type: Some("application/pdf".into()), }, ); let (response, reads) = drive_native_file_call( request, - Ok(super::OcrFileContent { + Ok(crate::ocr::types::OcrFileContent { bytes: b"abc".as_slice().into(), file_name: Some("scan.png".into()), }), @@ -484,30 +492,27 @@ async fn host_reader_documents_are_read_once_at_the_core_selected_point_and_enco async fn host_reader_failures_and_empty_files_fail_before_the_provider_is_called() { let (base, seen, _server) = mock_server(vec![]).await; let request = wire_request("mistral/model", &base, json!({})); - let failure = crate::ocr::Error::InvalidRequest("reader exploded".into()); + let failure = OcrError::InvalidRequest("reader exploded".into()); let (response, reads) = drive_native_file_call( - request.with_document(super::OcrDocumentInput::HostReader { mime_type: None }), + request.with_document(crate::ocr::types::OcrDocumentInput::HostReader { mime_type: None }), Err(failure.clone()), ) .await; assert!( - matches!(response.unwrap_err(), crate::ocr::Error::InvalidRequest(message) if message == "reader exploded") + matches!(response.unwrap_err(), OcrError::InvalidRequest(message) if message == "reader exploded") ); assert_eq!(reads, 1); let request = wire_request("mistral/model", &base, json!({})); let (response, _) = drive_native_file_call( - request.with_document(super::OcrDocumentInput::HostReader { mime_type: None }), - Ok(super::OcrFileContent { + request.with_document(crate::ocr::types::OcrDocumentInput::HostReader { mime_type: None }), + Ok(crate::ocr::types::OcrFileContent { bytes: Default::default(), file_name: None, }), ) .await; - assert!(matches!( - response.unwrap_err(), - crate::ocr::Error::EmptyFile - )); + assert!(matches!(response.unwrap_err(), OcrError::EmptyFile)); assert!(seen.lock().unwrap().is_empty()); } @@ -522,16 +527,13 @@ async fn path_documents_are_read_by_core_without_a_host_operation() { let path = dir.join("scan.png"); std::fs::write(&path, b"abc").unwrap(); let request = wire_request("mistral/model", &base, json!({})).with_document( - super::OcrDocumentInput::Path { + crate::ocr::types::OcrDocumentInput::Path { path: path.clone(), mime_type: None, }, ); - let (response, reads) = drive_native_file_call( - request, - Err(crate::ocr::Error::InvalidRequest("unused".into())), - ) - .await; + let (response, reads) = + drive_native_file_call(request, Err(OcrError::InvalidRequest("unused".into()))).await; server.await.unwrap(); std::fs::remove_dir_all(&dir).unwrap(); assert_eq!(response.unwrap().pages[0].markdown, "path"); @@ -541,16 +543,16 @@ async fn path_documents_are_read_by_core_without_a_host_operation() { let (base, seen, _server) = mock_server(vec![]).await; let request = wire_request("mistral/model", &base, json!({})); let (response, _) = drive_native_file_call( - request.with_document(super::OcrDocumentInput::Path { + request.with_document(crate::ocr::types::OcrDocumentInput::Path { path: path.clone(), mime_type: None, }), - Err(crate::ocr::Error::InvalidRequest("unused".into())), + Err(OcrError::InvalidRequest("unused".into())), ) .await; assert!(matches!( response.unwrap_err(), - crate::ocr::Error::FileRead { path: failed, source } if failed == path && source.kind() == std::io::ErrorKind::NotFound + OcrError::FileRead { path: failed, source } if failed == path && source.kind() == std::io::ErrorKind::NotFound )); assert!(seen.lock().unwrap().is_empty()); } @@ -563,14 +565,12 @@ async fn cancellation_at_before_send_prevents_execution_and_further_resumption() json!({}), )); let (outcome, ops, mut machine) = drive_until(ocr_client(), &host, |_| { - Err(HostFailure::Cancelled(crate::ocr::Error::InvalidRequest( + Err(HostFailure::Cancelled(OcrError::InvalidRequest( "cancelled".into(), ))) }) .await; - assert!( - matches!(outcome, Err(crate::ocr::Error::InvalidRequest(message)) if message == "cancelled") - ); + assert!(matches!(outcome, Err(OcrError::InvalidRequest(message)) if message == "cancelled")); assert_eq!(ops, ["ProjectRequest", "BeforeSend"]); assert!(machine.resume(Some(HostResult::Emitted)).await.is_err()); } @@ -596,10 +596,7 @@ async fn missing_host_result_preserves_pending_operation() { )); } -async fn read_bounded_response( - response: Vec, - limit: usize, -) -> Result { +async fn read_bounded_response(response: Vec, limit: usize) -> Result { use tokio::io::{AsyncReadExt, AsyncWriteExt}; let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); @@ -618,7 +615,7 @@ async fn read_bounded_response( .unwrap(); let result = tokio::time::timeout( std::time::Duration::from_secs(2), - super::client::read_response_bytes(response, limit), + litellm_llms::custom_httpx::llm_http_handler::read_response_bytes(response, limit), ) .await; server.abort(); @@ -628,7 +625,7 @@ async fn read_bounded_response( #[tokio::test] async fn response_limit_accepts_exact_size_and_rejects_declared_and_chunked_overflow() { - use super::Error; + use litellm_llms::base_llm::ocr::error::Error; for response in [ "HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nabcdefgh", @@ -670,7 +667,10 @@ async fn oversized_error_retains_http_status_and_bounded_diagnostics_without_dra .await .unwrap_err(); match error { - super::Error::Transport(crate::transport::Error::Http { status, body }) => { + OcrError::Transport(litellm_llms::custom_httpx::transport::Error::Http { + status, + body, + }) => { assert_eq!(status, 429); assert_eq!(body, prefix); } @@ -693,7 +693,7 @@ fn response_limit_is_validated_and_not_forwarded_to_the_provider() { json!(true), json!("123"), json!(1.5), - json!(crate::constants::OCR_RESPONSE_MAX_BYTES + 1), + json!(OCR_RESPONSE_MAX_BYTES + 1), Value::Null, ] { let wire = serde_json::from_value(json!({ @@ -738,8 +738,8 @@ async fn interrupt_drops_provider_captures_before_returning() { let entered = Arc::new(tokio::sync::Notify::new()); let dropped = Arc::new(AtomicBool::new(false)); let request = wire_request("azure_ai/mistral-ocr", "https://example.invalid", json!({})); - let request = super::LiteLLMOcrRequest { - transport: super::OcrTransportConfig { + let request = crate::ocr::types::LiteLLMOcrRequest { + transport: OcrTransportConfig { extra_headers: vec![("authorization".into(), "Bearer test-key".into())], ..request.transport }, @@ -774,24 +774,24 @@ async fn interrupt_drops_provider_captures_before_returning() { .await .unwrap(); assert!(!dropped.load(Ordering::SeqCst)); - let selected = crate::ocr::Error::InvalidRequest("cancelled".into()); + let selected = OcrError::InvalidRequest("cancelled".into()); let acknowledgement = machine.interrupt(HostFailure::Cancelled(selected.clone())); assert!( dropped.load(Ordering::SeqCst), "interrupt returned while provider captures were still alive" ); assert!( - matches!(acknowledgement.await, Err(crate::ocr::Error::InvalidRequest(message)) if message == "cancelled") + matches!(acknowledgement.await, Err(OcrError::InvalidRequest(message)) if message == "cancelled") ); } struct CallerTokenHost { - request: Mutex>, + request: Mutex>, trace: Mutex>, } -impl Host for CallerTokenHost { - async fn route(&self, op: OcrOp) -> Result { +impl Host for CallerTokenHost { + async fn route(&self, op: OcrOp) -> Result { match op { OcrOp::ProjectRequest => { self.trace.lock().unwrap().push("project".into()); @@ -808,7 +808,7 @@ impl Host for CallerTokenHost { )), )) } - OcrOp::ReadDocument => Err(crate::ocr::Error::InvalidRequest("no reader".into())), + OcrOp::ReadDocument => Err(OcrError::InvalidRequest("no reader".into())), } } @@ -816,7 +816,7 @@ impl Host for CallerTokenHost { &self, wire: WireRequest, _: &litellm_callbacks::event::RequestContext, - ) -> Result { + ) -> Result { let is_authorization = |name: &str| name.eq_ignore_ascii_case("authorization"); let authorization = wire .headers @@ -910,7 +910,7 @@ async fn interrupting_an_in_flight_provider_request_closes_its_connection() { .await .unwrap(); - let cancelled = crate::ocr::Error::InvalidRequest("cancelled".into()); + let cancelled = OcrError::InvalidRequest("cancelled".into()); assert!( machine .interrupt(HostFailure::Cancelled(cancelled)) diff --git a/litellm-rust/crates/core/tests/ocr/passthrough.rs b/litellm-rust/crates/core/tests/ocr/passthrough.rs index c1cd1adf291..0273b48664d 100644 --- a/litellm-rust/crates/core/tests/ocr/passthrough.rs +++ b/litellm-rust/crates/core/tests/ocr/passthrough.rs @@ -1,16 +1,19 @@ -use std::collections::BTreeSet; -use std::sync::{Arc, Mutex}; +use std::{ + collections::BTreeSet, + sync::{Arc, Mutex}, +}; use litellm_callbacks::event::{RequestContext, WireRequest}; +use litellm_llms::base_llm::ocr::error::Error; use rstest::rstest; use rstest_reuse::{self, apply, template}; use serde_json::{Map, Value, json}; -use super::LocalOcrHost; use super::test_support::{ MockResponse, SERVED_DOCUMENT, document_server, mock_server, perform_ocr_with, request_body, wire_request_with_document, }; +use crate::ocr::route::LocalOcrHost; #[derive(Clone, Copy, Debug)] enum Route { @@ -107,7 +110,7 @@ impl Host { struct Sent { caller: Map, - result: Result<(), crate::ocr::Error>, + result: Result<(), Error>, before_send: Option<(WireRequest, RequestContext)>, provider_body: Option, } diff --git a/litellm-rust/crates/core/tests/ocr/support.rs b/litellm-rust/crates/core/tests/ocr/support.rs index 224a9d9e8f9..f3adf27cfa6 100644 --- a/litellm-rust/crates/core/tests/ocr/support.rs +++ b/litellm-rust/crates/core/tests/ocr/support.rs @@ -1,5 +1,11 @@ use std::sync::{Arc, Mutex}; +use futures_util::future::BoxFuture; +use litellm_callbacks::event::{Passthrough, WireRequest}; +use litellm_llms::{ + base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse}, + custom_httpx::llm_http_handler::{CallHooks, OcrClient}, +}; use serde_json::{Value, json}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, @@ -7,10 +13,29 @@ use tokio::{ }; use crate::ocr::{ - LiteLLMOcrRequest, LiteLLMOcrResponse, LocalOcrHost, OcrClient, ocr_machine, + route::{LocalOcrHost, ocr_machine}, + types::LiteLLMOcrRequest, wire::{OcrWireRequest, decode_request}, }; +/// Stands in for a host with no hooks registered: the wire request goes out unchanged +/// and response events go nowhere. +pub(crate) struct NoHooks; + +impl CallHooks for NoHooks { + fn before_send( + &self, + wire: WireRequest, + _passthrough_fields: Passthrough, + ) -> BoxFuture<'_, Result> { + Box::pin(async move { Ok(wire) }) + } + + fn response_received<'a>(&'a self, _body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> { + Box::pin(async { Ok(()) }) + } +} + pub(crate) fn ocr_client() -> OcrClient { let document_http = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) @@ -19,15 +44,11 @@ pub(crate) fn ocr_client() -> OcrClient { OcrClient::for_test(reqwest::Client::new(), document_http) } -pub(crate) async fn perform_ocr( - request: LiteLLMOcrRequest, -) -> Result { - ocr_client().perform(request).await +pub(crate) async fn perform_ocr(request: LiteLLMOcrRequest) -> Result { + crate::ocr::client::perform(&ocr_client(), request).await } -pub(crate) async fn perform_ocr_with( - host: LocalOcrHost, -) -> Result { +pub(crate) async fn perform_ocr_with(host: LocalOcrHost) -> Result { litellm_callbacks::run::run(ocr_machine(ocr_client()), &host).await } diff --git a/litellm-rust/crates/core/tests/reducto_ocr.rs b/litellm-rust/crates/core/tests/reducto_ocr.rs index a4c2119664f..59891b16e90 100644 --- a/litellm-rust/crates/core/tests/reducto_ocr.rs +++ b/litellm-rust/crates/core/tests/reducto_ocr.rs @@ -1,11 +1,10 @@ use litellm_callbacks::event::{CallEvent, WireRequest}; +use litellm_llms::base_llm::ocr::{error::Error, transformation::OcrDocument}; use rstest::rstest; use serde_json::{Value, json}; -use super::{ - LocalOcrHost, - test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request}, -}; +use super::test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request}; +use crate::ocr::route::LocalOcrHost; fn request_body(request: &str) -> Value { serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap() @@ -85,8 +84,8 @@ async fn data_uri_upload_preserves_multipart_headers( } else { json!({"type":"document_url","document_url":format!("data:{mime_type};base64,YWJj")}) }; - let mut request = super::LiteLLMOcrRequest { - document: serde_json::from_value::(document) + let mut request = crate::ocr::types::LiteLLMOcrRequest { + document: serde_json::from_value::(document) .unwrap() .into(), ..wire_request(&format!("reducto/{model}"), &base, json!({})) @@ -185,17 +184,14 @@ async fn upload_failure_stops_before_parse() { } #[rstest] -#[case("https://example.com/a.pdf", crate::ocr::Error::ReductoSource)] -#[case("reducto://", crate::ocr::Error::RequestField { path: "document file id".into() })] -#[case("data:application/pdf;base64", crate::ocr::Error::InvalidDataUri)] -#[case( - "data:application/pdf;base64,INVALID!", - crate::ocr::Error::InvalidDataUri -)] +#[case("https://example.com/a.pdf", Error::ReductoSource)] +#[case("reducto://", Error::RequestField { path: "document file id".into() })] +#[case("data:application/pdf;base64", Error::InvalidDataUri)] +#[case("data:application/pdf;base64,INVALID!", Error::InvalidDataUri)] #[tokio::test] async fn rejects_invalid_document_sources_before_network( #[case] source: &str, - #[case] expected: super::Error, + #[case] expected: Error, ) { let (base, seen, server) = mock_server(vec![MockResponse::json(json!({}))]).await; let request = super::test_support::with_source( @@ -220,7 +216,7 @@ async fn rejects_invalid_document_sources_before_network( #[test] fn response_normalization_groups_blocks_and_distinguishes_null_result() { - use crate::llms::reducto::ocr::transformation::{ + use litellm_llms::reducto::ocr::transformation::{ ReductoResponse, normalize_response as transform_ocr_response, }; @@ -353,3 +349,236 @@ async fn guardrail_rewrites_document_before_upload() { assert!(requests[0].starts_with("POST /parse ")); assert!(requests[0].contains("reducto://guarded.pdf")); } + +mod transformation { + use litellm_callbacks::event::{CallEvent, WireRequest}; + use litellm_llms::{ + base_llm::ocr::transformation::{BaseOcrConfig, OcrConnection, OcrRequestContext}, + reducto::ocr::transformation::*, + }; + use rstest::rstest; + + use super::*; + use crate::ocr::{ + route::LocalOcrHost, + test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request}, + }; + + #[tokio::test] + async fn v3_options_preserve_explicit_null() { + let overrides = + serde_json::from_value(json!({"formatting":null,"settings":{},"unknown":true})) + .unwrap(); + let params = ReductoParseV3Config + .map_ocr_params(&overrides, "parse-v3") + .unwrap(); + let client = crate::ocr::test_support::ocr_client(); + let connection = OcrConnection::default(); + let document = serde_json::from_value( + json!({"type":"document_url","document_url":"reducto://ready.pdf"}), + ) + .unwrap(); + let body = ReductoParseV3Config + .async_transform_ocr_request( + "parse-v3", + document, + ¶ms, + &[], + OcrRequestContext { + client: &client, + connection: &connection, + }, + ) + .await + .unwrap(); + assert_eq!( + serde_json::to_value(body).unwrap(), + json!({ + "input":"reducto://ready.pdf", "formatting":null, "settings":{} + }) + ); + let absent = ReductoParseV3Config + .map_ocr_params( + &litellm_core_utils::call_arguments::CallArguments::default(), + "parse-v3", + ) + .unwrap(); + assert_eq!(serde_json::to_value(absent).unwrap(), json!({})); + } + + #[rstest] + #[case( + "reducto/parse-v3", + json!({ + "formatting":{"table_output_format":"html"}, + "retrieval":{"chunk_mode":"section"}, + "settings":{"ocr_system":"standard"}, + "future_ocr_option":true, + "extra_body":{"provider_option":"value"} + }), + "reducto://already.pdf", + json!({ + "input":"reducto://already.pdf", + "formatting":{"table_output_format":"html"}, + "retrieval":{"chunk_mode":"section"}, + "settings":{"ocr_system":"standard"}, + "future_ocr_option":true, + "provider_option":"value" + }) + )] + #[case( + "reducto/parse-legacy", + json!({ + "enhance":{"agentic":[{"type":"table"}]}, + "future_ocr_option":true, + "extra_body":{"provider_option":"value"} + }), + "reducto://legacy.pdf", + json!({ + "document_url":"reducto://legacy.pdf", + "options":{"enhance":{"agentic":[{"type":"table"}]}}, + "future_ocr_option":true, + "provider_option":"value" + }) + )] + #[tokio::test] + async fn request_mapping_matches_python( + #[case] model: &str, + #[case] options: Value, + #[case] source: &str, + #[case] expected: Value, + ) { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ + "result":{"chunks":[]} + }))]) + .await; + let request = + crate::ocr::test_support::with_source(wire_request(model, &base, options), source); + + perform_ocr(request).await.unwrap(); + server.await.unwrap(); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 1); + assert!(requests[0].starts_with("POST /parse ")); + assert_eq!(request_body(&requests[0]), expected); + } + + #[rstest] + #[case("parse-v3")] + #[case("parse-legacy")] + #[tokio::test] + async fn data_uri_upload_preserves_multipart_headers(#[case] model: &str) { + let (base, seen, server) = mock_server(vec![ + MockResponse::json(json!({"file_id":"reducto://uploaded.pdf"})), + MockResponse::json(json!({"result":{"chunks":[{"content":"hello"}]}})), + ]) + .await; + let mut request = wire_request(&format!("reducto/{model}"), &base, json!({})); + request.transport.extra_headers = vec![ + ("Content-Type".into(), "application/json".into()), + ("X-Trace".into(), "upload-test".into()), + ]; + + let response = perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!(response.pages[0].markdown, "hello"); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[0].starts_with("POST /upload ")); + assert!( + requests[0] + .to_ascii_lowercase() + .contains("content-type: multipart/form-data; boundary=") + ); + assert!(requests[0].contains("x-trace: upload-test")); + assert!(requests[0].contains("application/pdf")); + assert!(requests[0].contains("abc")); + assert!(requests[1].starts_with("POST /parse ")); + } + + #[tokio::test] + async fn response_received_stays_after_reducto_upload_and_parse() { + let (base, seen, server) = mock_server(vec![ + MockResponse::json(json!({"file_id":"reducto://uploaded.pdf"})), + MockResponse::json(json!({"result":{"chunks":[]}})), + ]) + .await; + let request_count = seen.clone(); + let host = LocalOcrHost::new(wire_request("reducto/parse-v3", &base, json!({}))) + .with_observer(move |event| { + if let CallEvent::ResponseReceived { raw } = event { + assert_eq!(request_count.lock().unwrap().len(), 2); + assert_eq!(raw.body, r#"{"result":{"chunks":[]}}"#); + } + }); + + perform_ocr_with(host).await.unwrap(); + server.await.unwrap(); + assert_eq!(seen.lock().unwrap().len(), 2); + } + + #[rstest] + #[case("https://example.com/a.pdf")] + #[case("reducto://")] + #[case("data:application/pdf;base64")] + #[case("data:application/pdf;base64,INVALID!")] + #[tokio::test] + async fn rejects_invalid_document_sources_before_network(#[case] source: &str) { + let request = crate::ocr::test_support::with_source( + wire_request("reducto/parse-v3", "http://127.0.0.1:1", json!({})), + source, + ); + assert!(perform_ocr(request).await.is_err()); + } + + #[tokio::test] + async fn facade_omits_native_response_by_default_and_preserves_auth_priority() { + let raw = json!({"job_id":"job-1","result":{"chunks":[]}}); + let (base, seen, server) = mock_server(vec![MockResponse::json(raw)]).await; + let mut request = crate::ocr::test_support::with_source( + wire_request("reducto/parse-v3", &base, json!({})), + "reducto://ready.pdf", + ); + request.transport.extra_headers = vec![("authorization".into(), "Bearer existing".into())]; + + let response = perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!(response.provider_native_response, None); + assert!( + seen.lock().unwrap()[0] + .to_ascii_lowercase() + .contains("authorization: bearer existing") + ); + } + + #[rstest] + #[case("reducto/parse-v3")] + #[case("reducto/parse-legacy")] + #[tokio::test] + async fn guardrail_headers_reach_upload_and_parse(#[case] model: &str) { + let (base, seen, server) = mock_server(vec![ + MockResponse::json(json!({"file_id":"reducto://uploaded.pdf"})), + MockResponse::json(json!({"result":{"chunks":[]}})), + ]) + .await; + let mut request = wire_request(model, &base, json!({})); + request.transport.extra_headers = vec![("authorization".into(), "Bearer original".into())]; + let host = LocalOcrHost::new(request).with_before_send(|wire, _| { + Ok(WireRequest { + headers: vec![("authorization".into(), "Bearer guarded".into())], + ..wire + }) + }); + + perform_ocr_with(host).await.unwrap(); + server.await.unwrap(); + let requests = seen.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert!(requests[0].starts_with("POST /upload ")); + assert!(requests[1].starts_with("POST /parse ")); + for request in requests.iter() { + assert!(request.contains("authorization: Bearer guarded")); + assert!(!request.contains("Bearer original")); + } + } +} diff --git a/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs b/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs index 6be30f784c4..2e8d69f5f64 100644 --- a/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs +++ b/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs @@ -56,11 +56,11 @@ async fn facade_executes_vertex_deepseek_at_the_openai_endpoint() { #[test] fn host_registration_selects_deepseek_without_affecting_mistral() { - assert!(crate::ocr::wire::is_supported_request( + assert!(crate::ocr::arguments::is_supported_request( "deepseek-ocr-maas", Some("vertex_ai") )); - assert!(crate::ocr::wire::is_supported_request( + assert!(crate::ocr::arguments::is_supported_request( "mistral-ocr-maas", Some("vertex_ai") )); @@ -85,3 +85,59 @@ async fn request_controlled_api_base_is_rejected_before_vertex_auth() { .contains("request-controlled Vertex AI endpoint") ); } + +mod deepseek_transformation { + use serde_json::json; + + use super::*; + use crate::ocr::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; + + #[tokio::test] + async fn facade_executes_vertex_deepseek_at_the_openai_endpoint() { + let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ + "choices":[{"message":{"content":"recognized"}}], + "usage":{"prompt_tokens":1} + }))]) + .await; + let request = wire_request( + "vertex_ai/deepseek-ocr-maas", + &base, + json!({ + "vertex_project":"project-1", + "vertex_location":"europe-west4", + "temperature":0.1, + "future_ocr_option":true, + "extra_body":{"provider_option":"value"} + }), + ); + let request = crate::ocr::test_support::with_source(request, "gs://bucket/document.pdf"); + + let response = perform_ocr(request).await.unwrap(); + server.await.unwrap(); + assert_eq!(response.pages[0].markdown, "recognized"); + assert_eq!( + response.usage_info.unwrap().extra_fields["prompt_tokens"], + 1 + ); + let requests = seen.lock().unwrap(); + assert!(requests[0].starts_with( + "POST /v1/projects/project-1/locations/europe-west4/endpoints/openapi/chat/completions " + )); + assert!( + requests[0] + .to_ascii_lowercase() + .contains("authorization: bearer test-key") + ); + let body = request_body(&requests[0]); + assert_eq!(body["model"], "deepseek-ai/deepseek-ocr-maas"); + assert_eq!(body["temperature"], 0.1); + assert_eq!(body["future_ocr_option"], true); + assert_eq!(body["provider_option"], "value"); + assert!(body.get("vertex_project").is_none()); + assert!(body.get("extra_body").is_none()); + assert_eq!( + body["messages"][0]["content"][0], + json!({"type":"image_url","image_url":"gs://bucket/document.pdf"}) + ); + } +} diff --git a/litellm-rust/crates/core/tests/vertex_ai_ocr.rs b/litellm-rust/crates/core/tests/vertex_ai_ocr.rs index 9cd735c26dd..1f1186c7827 100644 --- a/litellm-rust/crates/core/tests/vertex_ai_ocr.rs +++ b/litellm-rust/crates/core/tests/vertex_ai_ocr.rs @@ -1,4 +1,5 @@ use litellm_auth::InputSource; +use litellm_llms::base_llm::ocr::transformation::OcrResponseFormat; use serde_json::{Value, json}; use super::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; @@ -102,15 +103,14 @@ async fn request_controlled_api_base_is_rejected_before_vertex_auth() { async fn adapters_build_complete_requests_and_share_mistral_normalization() { use std::time::Duration; - use crate::{ - llms::{ - base_llm::ocr::transformation::BaseOcrConfig, - mistral::ocr::transformation::MistralOcrConfig, - vertex_ai::ocr::transformation::VertexAiOcrConfig, - }, - ocr::test_support::ocr_client, + use litellm_llms::{ + base_llm::ocr::transformation::BaseOcrConfig, + mistral::ocr::transformation::MistralOcrConfig, + vertex_ai::ocr::transformation::VertexAiOcrConfig, }; + use crate::ocr::test_support::ocr_client; + let client = ocr_client(); let options = json!({ "pages": [0, 2], @@ -132,11 +132,11 @@ async fn adapters_build_complete_requests_and_share_mistral_normalization() { super::test_support::resolved_request(vertex), ); let direct_http = MistralOcrConfig - .prepare_request(&direct, &client) + .prepare_request(&direct, &client, &crate::ocr::test_support::NoHooks) .await .unwrap(); let vertex_http = VertexAiOcrConfig - .prepare_request(&vertex, &client) + .prepare_request(&vertex, &client, &crate::ocr::test_support::NoHooks) .await .unwrap(); assert_eq!(direct_http.url().as_str(), "https://mistral.test/v1/ocr"); @@ -164,19 +164,11 @@ async fn adapters_build_complete_requests_and_share_mistral_normalization() { let payload = json!({"pages": [{"index": 0, "markdown": "hello"}], "extra": "preserved"}); let raw = serde_json::to_vec(&payload).unwrap(); let direct_response = MistralOcrConfig - .transform_ocr_response( - &direct.model, - &raw, - crate::ocr::types::OcrResponseFormat::Litellm, - ) + .transform_ocr_response(&direct.model, &raw, OcrResponseFormat::Litellm) .unwrap() .into_json(); let vertex_response = VertexAiOcrConfig - .transform_ocr_response( - &vertex.model, - &raw, - crate::ocr::types::OcrResponseFormat::Litellm, - ) + .transform_ocr_response(&vertex.model, &raw, OcrResponseFormat::Litellm) .unwrap() .into_json(); assert_eq!(direct_response, vertex_response); @@ -184,3 +176,99 @@ async fn adapters_build_complete_requests_and_share_mistral_normalization() { assert_eq!(direct_response["object"], "ocr"); assert_eq!(direct_response["extra"], "preserved"); } + +mod transformation { + + use rstest::rstest; + use serde_json::{Value, json}; + + use crate::ocr::test_support::wire_request; + + #[rstest] + #[case::mistral(false)] + #[case::vertex(true)] + #[tokio::test] + async fn configs_build_complete_requests_and_share_mistral_normalization( + #[case] use_vertex: bool, + ) { + use std::time::Duration; + + use litellm_llms::{ + base_llm::ocr::transformation::BaseOcrConfig, + mistral::ocr::transformation::MistralOcrConfig, + vertex_ai::ocr::transformation::VertexAiOcrConfig, + }; + + use crate::ocr::test_support::ocr_client; + + let client = ocr_client(); + let options = json!({ + "pages": [0, 2], + "include_image_base64": true, + "vertex_project": "project-1", + "vertex_location": "us-central1", + "unknown": "preserved" + }); + let direct = wire_request( + "mistral/mistral-ocr-maas", + "https://mistral.test", + options.clone(), + ); + let vertex = wire_request("vertex_ai/mistral-ocr-maas", "https://vertex.test", options); + let direct = crate::ocr::prepare::prepare_request_for_test( + crate::ocr::test_support::resolved_request(direct), + ); + let vertex = crate::ocr::prepare::prepare_request_for_test( + crate::ocr::test_support::resolved_request(vertex), + ); + let direct_http = MistralOcrConfig + .prepare_request(&direct, &client, &crate::ocr::test_support::NoHooks) + .await + .unwrap(); + let vertex_http = VertexAiOcrConfig + .prepare_request(&vertex, &client, &crate::ocr::test_support::NoHooks) + .await + .unwrap(); + assert_eq!(direct_http.url().as_str(), "https://mistral.test/v1/ocr"); + assert_eq!( + vertex_http.url().as_str(), + "https://vertex.test/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict" + ); + let http = if use_vertex { + &vertex_http + } else { + &direct_http + }; + assert_eq!(http.method(), reqwest::Method::POST); + assert_eq!(http.headers()["authorization"], "Bearer test-key"); + assert_eq!(http.headers()["content-type"], "application/json"); + assert_eq!(http.timeout(), Some(&Duration::from_secs(2))); + let body: Value = serde_json::from_slice(http.body().unwrap().as_bytes().unwrap()).unwrap(); + assert_eq!( + body, + json!({ + "model": "mistral-ocr-maas", + "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, + "pages": [0, 2], + "include_image_base64": true, + "unknown": "preserved" + }) + ); + let payload = serde_json::to_vec( + &json!({"pages": [{"index": 0, "markdown": "hello"}], "extra": "preserved"}), + ) + .unwrap(); + let direct_response = MistralOcrConfig + .transform_ocr_response(&direct.model, &payload, Default::default()) + .unwrap() + .into_json(); + let vertex_response = VertexAiOcrConfig + .transform_ocr_response(&vertex.model, &payload, Default::default()) + .unwrap() + .into_json(); + assert_eq!(direct_response, vertex_response); + assert_eq!(direct_response["model"], "mistral-ocr-maas"); + assert_eq!(direct_response["object"], "ocr"); + assert_eq!(direct_response["extra"], "preserved"); + } +} diff --git a/litellm-rust/crates/llms/AGENTS.md b/litellm-rust/crates/llms/AGENTS.md new file mode 100644 index 00000000000..09fe20cd9d6 --- /dev/null +++ b/litellm-rust/crates/llms/AGENTS.md @@ -0,0 +1,21 @@ +litellm-llms mirrors `litellm/llms/`: base config traits, provider transformations, and the `custom_httpx` handlers. See `../core/AGENTS.md` for how the crates layer. + +## Python/Rust transformation pairs + +Use the base OCR and Mistral OCR pairs as the reference when aligning transformations. Derive `src/.rs` from `litellm/llms/.py`, preserving meaningful basenames such as `messages_transformation` + +Keep corresponding operation names and parameter names when their responsibilities match. Rust types retain the Python semantic name with Rust acronym casing (`BaseOCRConfig` / `BaseOcrConfig`, `MistralOCRConfig` / `MistralOcrConfig`). Private Python helpers can drop their leading underscore. Give Rust adapter helpers distinct responsibility names rather than duplicating trait method names + +Order OCR config methods as supported parameters, credential metadata and connection resolution, health-check input, parameter mapping, environment validation, URL construction, request transformation, async request transformation, response transformation, async response transformation, and error conversion. Put constants and data types before the config, private helpers after it in operation order, and tests last. Rust-only trait hooks follow the corresponding Python methods + +Use trait defaults for unchanged inherited behavior and explicit delegation for shared provider behavior. Keep typed inputs, ownership, `Result`, and async I/O idiomatic. A matching path or symbol identifies the counterpart, not a claim of full behavioral parity + +Use named `#[rstest]` cases for independent input/output scenarios instead of loops or repeated calls in one test. Inject reusable setup with `#[fixture]` arguments and use `#[with(...)]` for fixture overrides. Keep assertions about the same result together + +For base OCR, Python response models live next to `BaseOcrConfig` in `src/base_llm/ocr/transformation.rs`, as they do in Python; Rust context/environment types support the runtime. `BaseOcrConfig::prepare_request` corresponds to Python's HTTP-handler preparation rather than a `BaseOCRConfig` method, and `validate_request_body` is a Rust-only hook. `src/base_llm/ocr/error.rs` and `src/base_llm/ocr/document.rs` are Rust-only: the OCR error taxonomy shared with the route, and inline-document helpers shared by several providers + +For Mistral, `async_transform_ocr_request` uses the base default in both languages. `resolve_headers` and `build_ocr_url` implement the respective environment and URL operations, and `normalize_response` implements the typed part of response transformation. Existing auth key/header handling and top-level response-extra preservation differ between languages; layout refactors must preserve those behaviors and verify them with the existing tests + +For non-OCR pairs, order corresponding methods as parameter support/mapping, environment validation, URL construction, request transformation, and response transformation, followed by Rust-only runtime hooks. Auth resolution remains split between configs and route preparation in litellm-core. Chat `supported_openai_param_mappings` describes accepted OpenAI/provider name pairs, unlike Python's `get_supported_openai_params` name list. Audio `map_transcription_params` remains a Rust filtering helper + +Azure Messages maps to `llms/azure_ai/anthropic/messages_transformation.py`; Bedrock Converse maps to `llms/bedrock/chat/converse_transformation.py`. `AnthropicConfig`, `AmazonConverseConfig`, and the non-OCR base traits are partial ports. `OpenAiResponsesApiConfig` currently implements only the WebSocket surface. Preserve their acceptance gates, passthrough behavior, and host fallback contracts when aligning layout diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml new file mode 100644 index 00000000000..4ca6c7cb2a5 --- /dev/null +++ b/litellm-rust/crates/llms/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "litellm-llms" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[features] +test-support = [] + +[dependencies] +litellm-types.workspace = true +litellm-core-utils.workspace = true +litellm-auth.workspace = true +litellm-auth-aws.workspace = true +litellm-auth-azure.workspace = true +litellm-auth-gcp.workspace = true +litellm-callbacks.workspace = true +litellm-framing.workspace = true +base64.workspace = true +bytes.workspace = true +data-url = "0.3.2" +futures-util.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_json = { workspace = true, features = ["preserve_order"] } +serde_path_to_error = "0.1" +serde_with.workspace = true +thiserror.workspace = true +time.workspace = true +tokio = { workspace = true, features = ["sync"] } +url.workspace = true + +[dev-dependencies] +aws-smithy-eventstream = "=0.61.1" +aws-smithy-types = "1.6.1" +rstest.workspace = true +tokio.workspace = true diff --git a/litellm-rust/crates/core/src/llms/openai/responses/mod.rs b/litellm-rust/crates/llms/src/anthropic/batches/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/llms/openai/responses/mod.rs rename to litellm-rust/crates/llms/src/anthropic/batches/mod.rs diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs b/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs similarity index 97% rename from litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs rename to litellm-rust/crates/llms/src/anthropic/batches/transformation.rs index 8a314bd3e56..94e4dc7838a 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/batches.rs +++ b/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs @@ -1,10 +1,13 @@ -use litellm_providers::anthropic::experimental_pass_through::messages::transformation::resolve_anthropic_api_base; +use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; use serde::{Deserialize, Serialize}; use serde_json::Value; use time::OffsetDateTime; use url::Url; -use crate::messages::{Error, types::AnthropicMessagesResponse}; +use crate::{ + anthropic::experimental_pass_through::messages::transformation::resolve_anthropic_api_base, + base_llm::chat::transformation::Error, +}; const BATCHES_PATH_SUFFIX: &str = "/v1/messages/batches"; diff --git a/litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs b/litellm-rust/crates/llms/src/anthropic/chat/handler.rs similarity index 89% rename from litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs rename to litellm-rust/crates/llms/src/anthropic/chat/handler.rs index 2c540a4c436..a80cfbf28bd 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/chat/streaming.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/handler.rs @@ -1,18 +1,17 @@ use std::collections::HashMap; +use litellm_types::{ + llms::openai::{ChatCompletionThinkingBlock, ChatCompletionToolCallChunk}, + utils::{ChatCompletionChunk, ChatCompletionsUsage}, +}; use serde_json::Value; -use super::super::experimental_pass_through::messages::streaming::{ - AnthropicContentBlock, AnthropicContentBlockDelta, AnthropicMessagesStreamEvent, - AnthropicStreamUsage, -}; -use crate::chat_completions::{ - Error, - streaming::StreamTransformer, - types::{ - ChatCompletionChunk, ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, - ChatCompletionsUsage, +use crate::{ + anthropic::experimental_pass_through::messages::streaming_iterator::{ + AnthropicContentBlock, AnthropicContentBlockDelta, AnthropicMessagesStreamEvent, + AnthropicStreamUsage, }, + base_llm::{base_model_iterator::StreamTransformer, chat::transformation::Error}, }; #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/litellm-rust/crates/llms/src/anthropic/chat/mod.rs b/litellm-rust/crates/llms/src/anthropic/chat/mod.rs new file mode 100644 index 00000000000..f0050b7dc71 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/chat/mod.rs @@ -0,0 +1,2 @@ +pub mod handler; +pub mod transformation; diff --git a/litellm-rust/crates/providers/src/anthropic/chat/tests.rs b/litellm-rust/crates/llms/src/anthropic/chat/tests.rs similarity index 99% rename from litellm-rust/crates/providers/src/anthropic/chat/tests.rs rename to litellm-rust/crates/llms/src/anthropic/chat/tests.rs index 18b6efb13fd..40e89c52c3c 100644 --- a/litellm-rust/crates/providers/src/anthropic/chat/tests.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/tests.rs @@ -1,7 +1,7 @@ use serde_json::json; use super::*; -use crate::chat::Error; +use crate::base_llm::chat::transformation::Error; fn messages(value: Value) -> Vec { serde_json::from_value(value).expect("valid messages") diff --git a/litellm-rust/crates/providers/src/anthropic/chat/transformation.rs b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs similarity index 91% rename from litellm-rust/crates/providers/src/anthropic/chat/transformation.rs rename to litellm-rust/crates/llms/src/anthropic/chat/transformation.rs index 5288eebbb2f..21fa4e9f82e 100644 --- a/litellm-rust/crates/providers/src/anthropic/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs @@ -1,18 +1,24 @@ +use litellm_core_utils::{ + core_helpers::{finish_reason_for, unix_now, usage_from_parts}, + prompt_templates::factory::{Conversation, build_conversation}, +}; +use litellm_types::{ + llms::openai::ChatMessage, + utils::{ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse}, +}; use serde_json::{Map, Value, json}; -use crate::anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX; -use crate::anthropic::experimental_pass_through::messages::transformation::{ - complete_anthropic_url, resolve_anthropic_api_key, -}; -use crate::base_llm::chat::transformation::{ - BaseConfig, ChatCompletionsAuth, Unsupported, unsupported_message, unsupported_param, -}; -use crate::chat::Error; -use crate::chat::conversation::{Conversation, build_conversation}; -use crate::chat::response_utils::{finish_reason_for, unix_now, usage_from_parts}; -use crate::chat::types::{ - ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, ChatMessage, - ProviderChatRequestData, ProviderChatResponseData, +use crate::{ + anthropic::{ + ANTHROPIC_OAUTH_TOKEN_PREFIX, + experimental_pass_through::messages::transformation::{ + complete_anthropic_url, resolve_anthropic_api_key, + }, + }, + base_llm::chat::transformation::{ + BaseConfig, ChatCompletionsAuth, Error, ProviderChatRequestData, ProviderChatResponseData, + Unsupported, unsupported_message, unsupported_param, + }, }; /// Anthropic parameter names, post `map_openai_params`, that the Rust path can diff --git a/litellm-rust/crates/providers/src/anthropic/chat/mod.rs b/litellm-rust/crates/llms/src/anthropic/count_tokens/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/anthropic/chat/mod.rs rename to litellm-rust/crates/llms/src/anthropic/count_tokens/mod.rs diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs b/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs similarity index 94% rename from litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs rename to litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs index 3e599f67eb3..a4d8c57ca4f 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/count_tokens.rs +++ b/litellm-rust/crates/llms/src/anthropic/count_tokens/transformation.rs @@ -1,13 +1,8 @@ +use litellm_types::llms::anthropic_messages::anthropic_request::{AnthropicMessage, SystemPrompt}; use serde::{Deserialize, Serialize}; use serde_json::Value; -use crate::{ - constants::ANTHROPIC_OAUTH_TOKEN_PREFIX, - messages::{ - Error, - types::{AnthropicMessage, SystemPrompt}, - }, -}; +use crate::{anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX, base_llm::chat::transformation::Error}; const COUNT_TOKENS_ENDPOINT: &str = "https://api.anthropic.com/v1/messages/count_tokens"; const TOKEN_COUNTING_BETA: &str = "token-counting-2024-11-01"; @@ -97,10 +92,10 @@ impl AnthropicCountTokensConfig for AnthropicCountTokensTransformation { #[cfg(test)] mod tests { + use litellm_types::llms::anthropic_messages::anthropic_request::MessageContent; use serde_json::{Map, json}; use super::*; - use crate::messages::types::MessageContent; fn message() -> AnthropicMessage { AnthropicMessage { diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs new file mode 100644 index 00000000000..481d98c4e9d --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs @@ -0,0 +1,2 @@ +pub mod streaming_iterator; +pub mod transformation; diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/streaming.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/streaming_iterator.rs similarity index 94% rename from litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/streaming.rs rename to litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/streaming_iterator.rs index 92a36265df7..35e7d5820b0 100644 --- a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/messages/streaming.rs +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/streaming_iterator.rs @@ -9,7 +9,19 @@ use litellm_framing::{ use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use crate::messages::Error; +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("stream framing failed: {0}")] + StreamFraming(String), + #[error("Anthropic SSE frame has no data")] + MissingStreamData, + #[error("Anthropic stream event is invalid: {0}")] + InvalidStreamEvent(String), + #[error("Bedrock event payload is invalid: {0}")] + InvalidBedrockPayload(String), + #[error("Bedrock event payload has invalid base64: {0}")] + InvalidBedrockBase64(String), +} #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct AnthropicStreamUsage { diff --git a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/transformation.rs similarity index 97% rename from litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/transformation.rs rename to litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/transformation.rs index beabe440269..c791749ac6d 100644 --- a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/transformation.rs @@ -1,5 +1,6 @@ -use crate::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig; -use crate::messages::Error; +use crate::base_llm::{ + anthropic_messages::transformation::BaseAnthropicMessagesConfig, chat::transformation::Error, +}; const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; const ANTHROPIC_API_BASE_ENV: &str = "ANTHROPIC_API_BASE"; diff --git a/litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/mod.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/llms/anthropic/experimental_pass_through/mod.rs rename to litellm-rust/crates/llms/src/anthropic/experimental_pass_through/mod.rs diff --git a/litellm-rust/crates/providers/src/anthropic/mod.rs b/litellm-rust/crates/llms/src/anthropic/mod.rs similarity index 74% rename from litellm-rust/crates/providers/src/anthropic/mod.rs rename to litellm-rust/crates/llms/src/anthropic/mod.rs index 38a59aa6e0d..d181ceaca3c 100644 --- a/litellm-rust/crates/providers/src/anthropic/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/mod.rs @@ -1,4 +1,6 @@ +pub mod batches; pub mod chat; +pub mod count_tokens; pub mod experimental_pass_through; pub const ANTHROPIC_OAUTH_TOKEN_PREFIX: &str = "sk-ant-oat"; diff --git a/litellm-rust/crates/providers/src/azure_ai/anthropic/messages_transformation.rs b/litellm-rust/crates/llms/src/azure_ai/anthropic/messages_transformation.rs similarity index 96% rename from litellm-rust/crates/providers/src/azure_ai/anthropic/messages_transformation.rs rename to litellm-rust/crates/llms/src/azure_ai/anthropic/messages_transformation.rs index a79b9038144..99f55f18afc 100644 --- a/litellm-rust/crates/providers/src/azure_ai/anthropic/messages_transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/anthropic/messages_transformation.rs @@ -1,15 +1,19 @@ +use litellm_types::llms::anthropic_messages::{ + anthropic_request::{ + AnthropicMessage, AnthropicMessagesRequest, ContentBlock, MessageContent, SystemPrompt, + }, + anthropic_response::AnthropicMessagesResponse, +}; use serde_json::{Map, Value}; -use crate::anthropic::experimental_pass_through::messages::transformation::{ - ANTHROPIC_MESSAGES_CONFIG, AnthropicMessagesConfig, non_empty, -}; -use crate::base_llm::anthropic_messages::transformation::{ - BaseAnthropicMessagesConfig, MessagesAuthStrategy, -}; -use crate::messages::Error; -use crate::messages::types::{ - AnthropicMessage, AnthropicMessagesRequest, AnthropicMessagesResponse, ContentBlock, - MessageContent, SystemPrompt, +use crate::{ + anthropic::experimental_pass_through::messages::transformation::{ + ANTHROPIC_MESSAGES_CONFIG, AnthropicMessagesConfig, non_empty, + }, + base_llm::{ + anthropic_messages::transformation::{BaseAnthropicMessagesConfig, MessagesAuthStrategy}, + chat::transformation::Error, + }, }; const AZURE_API_KEY_ENV: &str = "AZURE_API_KEY"; diff --git a/litellm-rust/crates/providers/src/azure_ai/anthropic/mod.rs b/litellm-rust/crates/llms/src/azure_ai/anthropic/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/azure_ai/anthropic/mod.rs rename to litellm-rust/crates/llms/src/azure_ai/anthropic/mod.rs diff --git a/litellm-rust/crates/providers/src/azure_ai/mod.rs b/litellm-rust/crates/llms/src/azure_ai/mod.rs similarity index 59% rename from litellm-rust/crates/providers/src/azure_ai/mod.rs rename to litellm-rust/crates/llms/src/azure_ai/mod.rs index e529997219e..fd55dc91cd8 100644 --- a/litellm-rust/crates/providers/src/azure_ai/mod.rs +++ b/litellm-rust/crates/llms/src/azure_ai/mod.rs @@ -1 +1,2 @@ pub mod anthropic; +pub mod ocr; diff --git a/litellm-rust/crates/core/src/llms/azure_ai/ocr/cohere_parse_transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs similarity index 79% rename from litellm-rust/crates/core/src/llms/azure_ai/ocr/cohere_parse_transformation.rs rename to litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs index 71ea7a279a6..f55f6b067e4 100644 --- a/litellm-rust/crates/core/src/llms/azure_ai/ocr/cohere_parse_transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs @@ -1,25 +1,23 @@ +use litellm_core_utils::{call_arguments::CallArguments, url_utils::ApiUrl}; use serde_json::Value; use crate::{ - call_arguments::CallArguments, - llms::{ - base_llm::ocr::transformation::{BaseOcrConfig, OcrRequestContext}, - cohere::ocr::{ - CohereOptions, - transformation::{CohereParseConfig, CohereRequest}, - validate_document, + base_llm::ocr::{ + document::{inline_remote_document, validate_inline_document}, + error::Error, + transformation::{ + BaseOcrConfig, LiteLLMOcrResponse, OcrDocument, OcrRequestContext, OcrResponseFormat, + PreparedOcrRequest, }, }, - ocr::{ - OcrClient, - document::{inline_remote_document, validate_inline_document}, - types::{LiteLLMOcrResponse, OcrDocument, PreparedOcrRequest}, + cohere::ocr::transformation::{ + CohereOptions, CohereParseConfig, CohereRequest, validate_document, }, - url_utils::ApiUrl, + custom_httpx::llm_http_handler::OcrClient, }; #[derive(Default)] -pub(crate) struct AzureAICohereParseConfig; +pub struct AzureAICohereParseConfig; impl BaseOcrConfig for AzureAICohereParseConfig { type OcrParams = CohereOptions; @@ -38,7 +36,7 @@ impl BaseOcrConfig for AzureAICohereParseConfig { &self, request: &PreparedOcrRequest, client: &OcrClient, - ) -> Result { + ) -> Result { BaseOcrConfig::validate_environment( &super::transformation::AzureAiOcrConfig, request, @@ -52,10 +50,10 @@ impl BaseOcrConfig for AzureAICohereParseConfig { request: &PreparedOcrRequest, _params: &Self::OcrParams, _environment: &Self::Environment, - ) -> Result { + ) -> Result { let base = super::transformation::AzureAiOcrConfig::resolve_api_base( request.connection.api_base.as_deref(), - &crate::ocr::prepare::credential_env, + &crate::base_llm::ocr::transformation::credential_env, )?; self.get_complete_url(&base) } @@ -66,7 +64,7 @@ impl BaseOcrConfig for AzureAICohereParseConfig { document: OcrDocument, params: &CohereOptions, headers: &[(String, String)], - ) -> Result { + ) -> Result { CohereParseConfig.transform_ocr_request(model, document, params, headers) } @@ -78,7 +76,7 @@ impl BaseOcrConfig for AzureAICohereParseConfig { &self, arguments: &CallArguments, model: &str, - ) -> Result { + ) -> Result { CohereParseConfig.map_ocr_params(arguments, model) } @@ -89,7 +87,7 @@ impl BaseOcrConfig for AzureAICohereParseConfig { optional_params: &CohereOptions, headers: &[(String, String)], context: OcrRequestContext<'_>, - ) -> Result { + ) -> Result { validate_document(&document)?; let document = inline_remote_document( context.client.document_fetcher(), @@ -104,20 +102,20 @@ impl BaseOcrConfig for AzureAICohereParseConfig { &self, model: &str, raw_response: &[u8], - request_format: crate::ocr::types::OcrResponseFormat, - ) -> Result { + request_format: OcrResponseFormat, + ) -> Result { CohereParseConfig.transform_ocr_response(model, raw_response, request_format) } - fn validate_request_body(&self, body: &Value) -> Result<(), crate::ocr::Error> { - let document = crate::ocr::prepare::body_document(body)?; + fn validate_request_body(&self, body: &Value) -> Result<(), Error> { + let document = crate::custom_httpx::llm_http_handler::body_document(body)?; validate_document(&document)?; validate_inline_document(&document) } } impl AzureAICohereParseConfig { - fn get_complete_url(&self, base: &str) -> Result { + fn get_complete_url(&self, base: &str) -> Result { let mut url = reqwest::Url::parse(base).map_err(|_| invalid_api_base())?; if !matches!(url.scheme(), "http" | "https") { return Err(invalid_api_base()); @@ -135,8 +133,8 @@ impl AzureAICohereParseConfig { } } -fn invalid_api_base() -> crate::ocr::Error { - crate::ocr::Error::RequestField { +fn invalid_api_base() -> Error { + Error::RequestField { path: "api_base".into(), } } diff --git a/litellm-rust/crates/core/src/llms/azure_ai/ocr/common_utils.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/common_utils.rs similarity index 87% rename from litellm-rust/crates/core/src/llms/azure_ai/ocr/common_utils.rs rename to litellm-rust/crates/llms/src/azure_ai/ocr/common_utils.rs index 4e7be1620ae..26eeeb6635c 100644 --- a/litellm-rust/crates/core/src/llms/azure_ai/ocr/common_utils.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/common_utils.rs @@ -3,12 +3,12 @@ use std::sync::OnceLock; use litellm_auth::{InputSource, Sourced}; use litellm_auth_azure::{AzureAuthInputs, AzureAuthService}; -use crate::ocr::types::OcrConnection; +use crate::base_llm::ocr::{error::Error, transformation::OcrConnection}; pub(super) async fn resolve_entra( config: &AzureAuthInputs, env_lookup: &(dyn Fn(&str) -> Option + Sync), -) -> Result>, crate::ocr::Error> { +) -> Result>, Error> { static SERVICE: OnceLock = OnceLock::new(); SERVICE .get_or_init(AzureAuthService::default) @@ -25,13 +25,13 @@ pub(super) async fn resolve_entra( Sourced::new(value, source) }) }) - .map_err(crate::ocr::Error::from) + .map_err(Error::from) } pub(super) fn validate_destination( connection: &OcrConnection, credential_source: InputSource, -) -> Result<(), crate::ocr::Error> { +) -> Result<(), Error> { if connection.api_base.is_some() && connection.api_base_source == InputSource::Request && credential_source != InputSource::Request diff --git a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/mod.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/anthropic/experimental_pass_through/messages/mod.rs rename to litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/mod.rs diff --git a/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs similarity index 54% rename from litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs rename to litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs index 7ad4b4d120f..87945bf8785 100644 --- a/litellm-rust/crates/core/src/llms/azure_ai/ocr/document_intelligence/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs @@ -3,6 +3,11 @@ use std::{collections::BTreeSet, time::Duration}; use base64::{Engine, engine::general_purpose::STANDARD}; use litellm_auth::{InputSource, Sourced}; use litellm_auth_azure::AzureAuthInputs; +use litellm_core_utils::{ + call_arguments::CallArguments, + serde_compat::{FiniteF64, LaxI64}, + url_utils::ApiUrl, +}; use reqwest::Url; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::{Map, Value}; @@ -10,36 +15,31 @@ use serde_with::serde_as; use tokio::time::Instant; use crate::{ - call_arguments::CallArguments, - constants::{ - AZURE_DI_API_VERSION, AZURE_DI_DEFAULT_DPI, AZURE_DI_DEFAULT_HEIGHT, - AZURE_DI_DEFAULT_WIDTH, AZURE_DI_SUBSCRIPTION_HEADER, OCR_POLL_RETRY_SECS, - }, - llms::base_llm::ocr::transformation::{ - BaseOcrConfig, OcrResponseContext, decode_and_normalize_response, - }, - ocr::{ - OcrClient, - client::read_json_response, + base_llm::ocr::{ document::InlineDocument, - json::DecodedOcrResponse, - prepare::credential_env, - route::OcrHost, - types::{ - LiteLLMOcrResponse, OcrConnection, OcrCredentialInputs, OcrDocument, OcrPage, - OcrPageDimensions, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest, - ResolvedOcrCredentials, + error::Error, + transformation::{ + BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES, + OCR_POLL_RETRY_SECS, OcrConnection, OcrCredentialInputs, OcrDocument, OcrPage, + OcrPageDimensions, OcrResponseContext, OcrResponseFormat, OcrUsageInfo, + PreparedOcrRequest, ResolvedOcrCredentials, credential_env, + decode_and_normalize_response, decode_response, }, }, - serde_compat::{FiniteF64, LaxI64}, - url_utils::ApiUrl, + custom_httpx::llm_http_handler::{CallHooks, OcrClient, read_json_response}, }; +const AZURE_DI_API_VERSION: &str = "2024-11-30"; +const AZURE_DI_SUBSCRIPTION_HEADER: &str = "Ocp-Apim-Subscription-Key"; +const AZURE_DI_DEFAULT_DPI: i64 = 96; +const AZURE_DI_DEFAULT_WIDTH: f64 = 8.5; +const AZURE_DI_DEFAULT_HEIGHT: f64 = 11.0; + const AZURE_DI_API_KEY_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_API_KEY"; const AZURE_DI_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT"; #[derive(Clone, Debug, PartialEq, Serialize)] -pub(crate) struct DocumentIntelligenceParams { +pub struct DocumentIntelligenceParams { #[serde(skip_serializing_if = "Option::is_none")] pub pages: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -48,7 +48,7 @@ pub(crate) struct DocumentIntelligenceParams { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(untagged)] -pub(crate) enum DocumentIntelligenceRequest { +pub enum DocumentIntelligenceRequest { UrlSource { #[serde(rename = "urlSource")] url_source: String, @@ -93,7 +93,7 @@ impl std::fmt::Display for OperationStatus { } #[derive(Clone, Debug, Deserialize)] -pub(crate) struct AzureDocumentIntelligenceOperation { +pub struct AzureDocumentIntelligenceOperation { status: Option, #[serde(rename = "analyzeResult")] analyze_result: Option, @@ -130,7 +130,7 @@ struct AzureDocumentIntelligenceLine { } #[derive(Clone, Debug)] -pub(crate) struct AzureDocumentIntelligenceOcrConfig; +pub struct AzureDocumentIntelligenceOcrConfig; impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { type OcrParams = DocumentIntelligenceParams; @@ -166,7 +166,7 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { &self, non_default_params: &CallArguments, _model: &str, - ) -> Result { + ) -> Result { Ok(DocumentIntelligenceParams { pages: normalize_pages_param(non_default_params.get("pages"))?, features: normalize_features_param(non_default_params.get("features"))?, @@ -177,7 +177,7 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { &self, request: &PreparedOcrRequest, _client: &OcrClient, - ) -> Result { + ) -> Result { let config = AzureAuthInputs { azure_ad_token_provider: request.azure_ad_token_provider.clone(), ..AzureAuthInputs::from_sourced_optional_params( @@ -194,10 +194,10 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { request: &PreparedOcrRequest, optional_params: &Self::OcrParams, _environment: &Self::Environment, - ) -> Result { + ) -> Result { let endpoint = nonblank(request.connection.api_base.clone()) .or_else(|| nonblank(credential_env(AZURE_DI_ENDPOINT_ENV))) - .ok_or_else(|| crate::ocr::Error::Auth(litellm_auth::Error::ProviderAuthentication("Missing Azure Document Intelligence API Base - Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT or pass api_base".into())))?; + .ok_or_else(|| Error::Auth(litellm_auth::Error::ProviderAuthentication("Missing Azure Document Intelligence API Base - Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT or pass api_base".into())))?; self.build_ocr_url(&endpoint, &request.model, optional_params) } @@ -207,7 +207,7 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { document: OcrDocument, _optional_params: &DocumentIntelligenceParams, _headers: &[(String, String)], - ) -> Result { + ) -> Result { build_request(document) } @@ -216,7 +216,7 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { model: &str, raw_response: &[u8], request_format: OcrResponseFormat, - ) -> Result { + ) -> Result { decode_and_normalize_response( model, raw_response, @@ -230,7 +230,7 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { model: &str, raw_response: reqwest::Response, context: OcrResponseContext<'_>, - ) -> Result { + ) -> Result { let decoded = read_operation_response( context.client.polling_http(), raw_response, @@ -238,7 +238,7 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { context.headers, context.connection, context.request_format == OcrResponseFormat::Native, - context.host, + context.hooks, ) .await?; Ok(LiteLLMOcrResponse { @@ -248,7 +248,7 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig { } } -fn normalize_pages_param(pages: Option<&Value>) -> Result, crate::ocr::Error> { +fn normalize_pages_param(pages: Option<&Value>) -> Result, Error> { let normalized = match pages { None | Some(Value::Null) => return Ok(None), Some(Value::Array(pages)) if pages.is_empty() => return Ok(None), @@ -257,12 +257,12 @@ fn normalize_pages_param(pages: Option<&Value>) -> Result, crate: .map(|page| { let page = page .as_i64() - .ok_or_else(|| crate::ocr::Error::Pages("page index is out of range".into()))?; + .ok_or_else(|| Error::Pages("page index is out of range".into()))?; if page < 0 { - return Err(crate::ocr::Error::Pages("negative page index".into())); + return Err(Error::Pages("negative page index".into())); } page.checked_add(1) - .ok_or_else(|| crate::ocr::Error::Pages("page index is out of range".into())) + .ok_or_else(|| Error::Pages("page index is out of range".into())) }) .collect::, _>>()? .into_iter() @@ -272,9 +272,10 @@ fn normalize_pages_param(pages: Option<&Value>) -> Result, crate: Some(Value::Array(tokens)) => tokens .iter() .map(|token| { - token.as_str().map(str::trim).ok_or_else(|| { - crate::ocr::Error::Pages("expected only integers or only strings".into()) - }) + token + .as_str() + .map(str::trim) + .ok_or_else(|| Error::Pages("expected only integers or only strings".into())) }) .collect::, _>>()? .join(","), @@ -284,13 +285,13 @@ fn normalize_pages_param(pages: Option<&Value>) -> Result, crate: .collect::>() .join(","), Some(_) => { - return Err(crate::ocr::Error::Pages( + return Err(Error::Pages( "expected an array of integers or strings, or a native page range".into(), )); } }; if !normalized.split(',').all(valid_page_token) { - return Err(crate::ocr::Error::Pages("invalid native page range".into())); + return Err(Error::Pages("invalid native page range".into())); } Ok(Some(normalized)) } @@ -311,15 +312,15 @@ fn valid_page_token(token: &str) -> bool { } } -fn normalize_features_param(features: Option<&Value>) -> Result, crate::ocr::Error> { +fn normalize_features_param(features: Option<&Value>) -> Result, Error> { let tokens = match features { None | Some(Value::Null) => return Ok(None), Some(Value::Array(names)) => names .iter() - .map(|name| name.as_str().ok_or(crate::ocr::Error::Features)) + .map(|name| name.as_str().ok_or(Error::Features)) .collect::, _>>()?, Some(Value::String(names)) => names.split(',').collect(), - Some(_) => return Err(crate::ocr::Error::Features), + Some(_) => return Err(Error::Features), }; if tokens.is_empty() { return Ok(None); @@ -331,20 +332,19 @@ fn normalize_features_param(features: Option<&Value>) -> Result, }; first.is_ascii_alphabetic() && rest.iter().all(u8::is_ascii_alphanumeric) }) { - return Err(crate::ocr::Error::Features); + return Err(Error::Features); } Ok(Some(normalized.join(","))) } -fn build_request(document: OcrDocument) -> Result { +fn build_request(document: OcrDocument) -> Result { let source = document.source(); if source.is_empty() { - return Err(crate::ocr::Error::MissingDocumentUrl); + return Err(Error::MissingDocumentUrl); } Ok(if let Some(document) = InlineDocument::parse(source)? { DocumentIntelligenceRequest::Base64Source { - base64_source: STANDARD - .encode(document.decode(crate::constants::OCR_INLINE_MAX_BYTES)?), + base64_source: STANDARD.encode(document.decode(OCR_INLINE_MAX_BYTES)?), } } else { DocumentIntelligenceRequest::UrlSource { @@ -356,9 +356,9 @@ fn build_request(document: OcrDocument) -> Result Result { +) -> Result { if response.status != Some(OperationStatus::Succeeded) { - return Err(crate::ocr::Error::OperationStatus( + return Err(Error::OperationStatus( response .status .map(|status| status.to_string()) @@ -371,8 +371,7 @@ fn transform_completed_response( .into_iter() .map(transform_azure_page) .collect::, _>>()?; - let pages_processed = - i64::try_from(pages.len()).map_err(|_| crate::ocr::Error::NumericRange("pages"))?; + let pages_processed = i64::try_from(pages.len()).map_err(|_| Error::NumericRange("pages"))?; Ok(LiteLLMOcrResponse { content: result.content, tables: result.tables, @@ -385,12 +384,12 @@ fn transform_completed_response( }) } -fn transform_azure_page(page: AzureDocumentIntelligencePage) -> Result { +fn transform_azure_page(page: AzureDocumentIntelligencePage) -> Result { let index = page .page_number .unwrap_or(1) .checked_sub(1) - .ok_or(crate::ocr::Error::NumericRange("page.pageNumber"))?; + .ok_or(Error::NumericRange("page.pageNumber"))?; let dimensions = convert_dimensions( page.width.unwrap_or(AZURE_DI_DEFAULT_WIDTH), page.height.unwrap_or(AZURE_DI_DEFAULT_HEIGHT), @@ -410,11 +409,7 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage) -> Result Result { +fn convert_dimensions(width: f64, height: f64, unit: &str) -> Result { let scale = if unit == "inch" { AZURE_DI_DEFAULT_DPI as f64 } else { @@ -427,10 +422,10 @@ fn convert_dimensions( }) } -fn pixel_dimension(value: f64, scale: f64, field: &'static str) -> Result { +fn pixel_dimension(value: f64, scale: f64, field: &'static str) -> Result { let value = value * scale; if !value.is_finite() || value < i64::MIN as f64 || value >= -(i64::MIN as f64) { - return Err(crate::ocr::Error::NumericRange(field)); + return Err(Error::NumericRange(field)); } Ok(value.trunc() as i64) } @@ -442,33 +437,38 @@ async fn read_operation_response( headers: &[(String, String)], connection: &OcrConnection, native: bool, - host: &OcrHost, -) -> Result, crate::ocr::Error> { + hooks: &dyn CallHooks, +) -> Result, Error> { if response.status() != reqwest::StatusCode::ACCEPTED { - let bytes = - crate::ocr::client::read_response_bytes(response, connection.max_response_bytes) - .await?; - crate::ocr::handler::emit_response_received(host, &bytes).await?; - return crate::ocr::json::decode_response(&bytes, native); + let bytes = crate::custom_httpx::llm_http_handler::read_response_bytes( + response, + connection.max_response_bytes, + ) + .await?; + hooks.response_received(&bytes).await?; + return decode_response(&bytes, native); } let location = response .headers() .get("operation-location") .and_then(|value| value.to_str().ok()) - .ok_or(crate::ocr::Error::PollLocation)? + .ok_or(Error::PollLocation)? .to_string(); - let original = Url::parse(original_url).map_err(|_| crate::ocr::Error::PollOrigin)?; - let operation = Url::parse(&location).map_err(|_| crate::ocr::Error::PollOrigin)?; + let original = Url::parse(original_url).map_err(|_| Error::PollOrigin)?; + let operation = Url::parse(&location).map_err(|_| Error::PollOrigin)?; if original.origin() != operation.origin() || !operation.username().is_empty() || operation.password().is_some() { - return Err(crate::ocr::Error::PollOrigin); + return Err(Error::PollOrigin); } - let bytes = - crate::ocr::client::read_response_bytes(response, connection.max_response_bytes).await?; - crate::ocr::handler::emit_response_received(host, &bytes).await?; - poll_operation(http_client, operation, headers, connection, native, host).await + let bytes = crate::custom_httpx::llm_http_handler::read_response_bytes( + response, + connection.max_response_bytes, + ) + .await?; + hooks.response_received(&bytes).await?; + poll_operation(http_client, operation, headers, connection, native, hooks).await } async fn poll_operation( @@ -477,29 +477,35 @@ async fn poll_operation( headers: &[(String, String)], connection: &OcrConnection, native: bool, - host: &OcrHost, -) -> Result, crate::ocr::Error> { + hooks: &dyn CallHooks, +) -> Result, Error> { let deadline = Instant::now() .checked_add(connection.poll_timeout) - .ok_or(crate::ocr::Error::PollTimeout)?; + .ok_or(Error::PollTimeout)?; loop { let remaining = deadline .checked_duration_since(Instant::now()) .filter(|remaining| !remaining.is_zero()) - .ok_or(crate::ocr::Error::PollTimeout)?; + .ok_or(Error::PollTimeout)?; let builder = http_client .get(url.clone()) .timeout(remaining.min(connection.timeout)); - let builder = crate::http_utils::with_headers( + let builder = crate::custom_httpx::http_handler::with_headers( builder, headers, - crate::http_utils::HeaderPolicy::Only(&[AZURE_DI_SUBSCRIPTION_HEADER, "authorization"]), + crate::custom_httpx::http_handler::HeaderPolicy::Only(&[ + AZURE_DI_SUBSCRIPTION_HEADER, + "authorization", + ]), ); - let response = tokio::time::timeout_at(deadline, crate::http_utils::http_request(builder)) - .await - .map_err(|_| crate::ocr::Error::PollTimeout)? - .map_err(crate::transport::Error::from)?; + let response = tokio::time::timeout_at( + deadline, + crate::custom_httpx::http_handler::http_request(builder), + ) + .await + .map_err(|_| Error::PollTimeout)? + .map_err(crate::custom_httpx::transport::Error::from)?; let retry = response .headers() .get(reqwest::header::RETRY_AFTER) @@ -516,19 +522,19 @@ async fn poll_operation( ), ) .await - .map_err(|_| crate::ocr::Error::PollTimeout)??; + .map_err(|_| Error::PollTimeout)??; match &decoded.data.status { Some(OperationStatus::Succeeded) => { - crate::ocr::handler::emit_response_received(host, decoded.text.as_bytes()).await?; + hooks.response_received(decoded.text.as_bytes()).await?; return Ok(decoded); } Some(OperationStatus::Running | OperationStatus::NotStarted) => { tokio::time::timeout_at(deadline, tokio::time::sleep(Duration::from_secs(retry))) .await - .map_err(|_| crate::ocr::Error::PollTimeout)?; + .map_err(|_| Error::PollTimeout)?; } status => { - return Err(crate::ocr::Error::OperationStatus( + return Err(Error::OperationStatus( status .as_ref() .map(ToString::to_string) @@ -545,7 +551,7 @@ impl AzureDocumentIntelligenceOcrConfig { endpoint: &str, model: &str, params: &DocumentIntelligenceParams, - ) -> Result { + ) -> Result { let model = format!("{}:analyze", model_id(model)?); ApiUrl::parse(endpoint) .and_then(|url| url.complete_path(&["documentintelligence", "documentModels", &model])) @@ -563,7 +569,7 @@ impl AzureDocumentIntelligenceOcrConfig { ) .into_string() }) - .map_err(|_| crate::ocr::Error::RequestField { + .map_err(|_| Error::RequestField { path: "api_base".into(), }) } @@ -573,9 +579,9 @@ impl AzureDocumentIntelligenceOcrConfig { connection: &OcrConnection, config: &AzureAuthInputs, env_lookup: &(dyn Fn(&str) -> Option + Sync), - ) -> Result, crate::ocr::Error> { - if crate::http_utils::has_header(&connection.extra_headers, "authorization") - || crate::http_utils::has_header( + ) -> Result, Error> { + if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization") + || crate::custom_httpx::http_handler::has_header( &connection.extra_headers, AZURE_DI_SUBSCRIPTION_HEADER, ) @@ -602,7 +608,7 @@ impl AzureDocumentIntelligenceOcrConfig { } let token = super::super::common_utils::resolve_entra(config, env_lookup) .await? - .ok_or(crate::ocr::Error::MissingAzureDocumentIntelligenceCredentials)?; + .ok_or(Error::MissingAzureDocumentIntelligenceCredentials)?; super::super::common_utils::validate_destination(connection, token.source())?; Ok( std::iter::once(("Authorization".into(), format!("Bearer {}", token.value()))) @@ -612,10 +618,10 @@ impl AzureDocumentIntelligenceOcrConfig { } } -fn model_id(model: &str) -> Result<&str, crate::ocr::Error> { +fn model_id(model: &str) -> Result<&str, Error> { let model = model.rsplit('/').next().unwrap_or(model); if matches!(model, "." | "..") { - return Err(crate::ocr::Error::DotModel); + return Err(Error::DotModel); } Ok(model) } @@ -633,7 +639,7 @@ mod tests { use super::*; - fn map(value: Value) -> Result { + fn map(value: Value) -> Result { let arguments = serde_json::from_value(value).unwrap(); AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, "model") } @@ -807,410 +813,4 @@ mod tests { (AZURE_DI_SUBSCRIPTION_HEADER.into(), "request-key".into()) ); } - - use std::sync::{Arc, Mutex}; - - use litellm_callbacks::event::CallEvent; - - use crate::ocr::{ - LocalOcrHost, - test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request}, - }; - - fn query_value(url: &str, key: &str) -> Option { - url::Url::parse(url) - .unwrap() - .query_pairs() - .find_map(|(name, value)| (name == key).then(|| value.into_owned())) - } - - #[tokio::test] - async fn facade_maps_pages_features_and_url_document() { - let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ - "status":"succeeded", - "analyzeResult":{"pages":[]} - }))]) - .await; - let mut request = wire_request( - "azure_ai/doc-intelligence/prebuilt-read", - &base, - json!({"pages":[2,0,0,1],"features":["keyValuePairs","languages"], "future_option": {"nested":null}, "extra_body":{"provider_option":false}}), - ); - request.document = serde_json::from_value::(json!({ - "type":"document_url", - "document_url":"https://example.com/document.pdf" - })) - .unwrap() - .into(); - - perform_ocr(request).await.unwrap(); - server.await.unwrap(); - let request = &seen.lock().unwrap()[0]; - let target = request.split_whitespace().nth(1).unwrap(); - let url = format!("{base}{target}"); - assert_eq!(query_value(&url, "pages").as_deref(), Some("1,2,3")); - assert_eq!( - query_value(&url, "features").as_deref(), - Some("keyValuePairs,languages") - ); - let body: Value = serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap(); - assert_eq!( - body, - json!({"urlSource":"https://example.com/document.pdf", "future_option":{"nested":null}, "provider_option":false}) - ); - } - - #[tokio::test] - async fn rejects_invalid_pages_features_and_format() { - for options in [ - json!({"pages":[true]}), - json!({"pages":[1,"2"]}), - json!({"pages":[-1]}), - json!({"pages":"1&&features=bad"}), - json!({"features":"languages&pages=1"}), - json!({"req_format":"azure"}), - ] { - let request = wire_request( - "azure_ai/doc-intelligence/prebuilt-read", - "http://127.0.0.1:1", - options.clone(), - ); - let rejected = perform_ocr(request).await.is_err(); - assert!(rejected, "accepted {options}"); - } - } - - #[tokio::test] - async fn inline_document_decodes_to_base64_source() { - let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ - "status":"succeeded" - }))]) - .await; - let request = wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({})); - - perform_ocr(request).await.unwrap(); - server.await.unwrap(); - let request = &seen.lock().unwrap()[0]; - let body: Value = serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap(); - assert_eq!(body, json!({"base64Source":"YWJj"})); - } - - #[tokio::test] - async fn immediate_response_normalizes_pages_and_preserves_native() { - let operation = json!({ - "status":"succeeded", - "operationExtension":42, - "analyzeResult":{ - "content":"A\n\nB", - "tables":[{"cells":[]}], - "keyValuePairs":[{"key":{"content":"A"}}], - "pages":[{ - "pageNumber":"2", - "width":"8.5", - "height":11, - "unit":"inch", - "lines":[{"content":"A"},{"content":null},{"content":"B"}] - }] - } - }); - let (base, _, server) = mock_server(vec![MockResponse::json(operation.clone())]).await; - let result = perform_ocr(wire_request( - "azure_ai/doc-intelligence/prebuilt-read", - &base, - json!({"req_format":"native"}), - )) - .await - .unwrap(); - server.await.unwrap(); - - assert_eq!(result.pages[0].index, 1); - assert_eq!(result.pages[0].markdown, "A\n\nB"); - assert_eq!( - serde_json::to_value(&result.pages[0].dimensions).unwrap(), - json!({"width":816,"height":1056,"dpi":96}) - ); - assert_eq!(result.usage_info.as_ref().unwrap().pages_processed, Some(1)); - let serialized = result.clone().into_json(); - assert_eq!(serialized["content"], "A\n\nB"); - assert_eq!(serialized["tables"], json!([{"cells":[]}])); - assert_eq!( - serialized["keyValuePairs"], - json!([{"key":{"content":"A"}}]) - ); - assert!(serialized.get("key_value_pairs").is_none()); - assert_eq!( - result.provider_native_response.as_ref(), - operation.as_object() - ); - } - - #[tokio::test] - async fn accepted_response_polls_to_success_with_only_credentials() { - let operation = json!({"status":"succeeded","analyzeResult":{"pages":[]}}); - let (base, seen, server) = mock_server(vec![ - MockResponse { - status: 202, - headers: vec![("Operation-Location", "{base}/operation".into())], - body: json!({}), - }, - MockResponse { - status: 200, - headers: vec![("Retry-After", "0".into())], - body: json!({"status":"running"}), - }, - MockResponse::json(operation.clone()), - ]) - .await; - let mut request = wire_request( - "azure_ai/doc-intelligence/prebuilt-read", - &base, - json!({"req_format":"native"}), - ); - request - .transport - .extra_headers - .push(("X-Trace".into(), "initial-only".into())); - - let result = perform_ocr(request).await.unwrap(); - server.await.unwrap(); - assert_eq!( - result.provider_native_response.as_ref(), - operation.as_object() - ); - let requests = seen.lock().unwrap(); - assert_eq!(requests.len(), 3); - assert!(requests[0].to_ascii_lowercase().contains("x-trace:")); - for poll in &requests[1..] { - assert!(!poll.to_ascii_lowercase().contains("x-trace:")); - assert!( - poll.to_ascii_lowercase() - .contains("ocp-apim-subscription-key: test-key") - ); - } - } - - #[tokio::test] - async fn accepted_response_emits_response_received_for_submission_and_completed_poll() { - let (base, seen, server) = mock_server(vec![ - MockResponse { - status: 202, - headers: vec![("Operation-Location", "{base}/operation".into())], - body: json!({"submitted": true}), - }, - MockResponse::json(json!({"status":"succeeded"})), - ]) - .await; - let responses_received = Arc::new(Mutex::new(Vec::new())); - let request_count = seen.clone(); - let observed = responses_received.clone(); - let host = LocalOcrHost::new(wire_request( - "azure_ai/doc-intelligence/prebuilt-read", - &base, - json!({}), - )) - .with_observer(move |event| { - if let CallEvent::ResponseReceived { raw } = event { - observed - .lock() - .unwrap() - .push((request_count.lock().unwrap().len(), raw.body.clone())); - } - }); - - perform_ocr_with(host).await.unwrap(); - server.await.unwrap(); - assert_eq!(seen.lock().unwrap().len(), 2); - assert_eq!( - *responses_received.lock().unwrap(), - [ - (1, r#"{"submitted":true}"#.to_string()), - (2, r#"{"status":"succeeded"}"#.to_string()), - ] - ); - } - - #[tokio::test] - async fn polling_forwards_bearer_credentials() { - let (base, seen, server) = mock_server(vec![ - MockResponse { - status: 202, - headers: vec![("Operation-Location", "{base}/operation".into())], - body: json!({}), - }, - MockResponse::json(json!({"status":"succeeded"})), - ]) - .await; - let mut request = wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({})); - request.credentials.api_key = None; - request.transport.extra_headers = vec![("Authorization".into(), "Bearer token".into())]; - - perform_ocr(request).await.unwrap(); - server.await.unwrap(); - let requests = seen.lock().unwrap(); - assert!( - requests[1] - .to_ascii_lowercase() - .contains("authorization: bearer token") - ); - } - - #[tokio::test] - async fn polling_does_not_follow_redirects() { - let (base, seen, server) = mock_server(vec![ - MockResponse { - status: 202, - headers: vec![("Operation-Location", "{base}/operation".into())], - body: json!({}), - }, - MockResponse { - status: 302, - headers: vec![("Location", "{base}/redirected".into())], - body: json!({}), - }, - MockResponse::json(json!({"status":"succeeded"})), - ]) - .await; - - let error = perform_ocr(wire_request( - "azure_ai/doc-intelligence/prebuilt-read", - &base, - json!({}), - )) - .await - .unwrap_err(); - - assert!(error.to_string().contains("status 302"), "{error}"); - assert_eq!(seen.lock().unwrap().len(), 2); - server.abort(); - } - - #[tokio::test] - async fn polling_rejects_terminal_failure() { - let (base, _, server) = mock_server(vec![ - MockResponse { - status: 202, - headers: vec![("Operation-Location", "{base}/operation".into())], - body: json!({}), - }, - MockResponse::json(json!({"status":"failed"})), - ]) - .await; - - let error = perform_ocr(wire_request( - "azure_ai/doc-intelligence/prebuilt-read", - &base, - json!({}), - )) - .await - .unwrap_err(); - server.await.unwrap(); - assert!(error.to_string().contains("status failed")); - } - - #[tokio::test] - async fn malformed_provider_pages_report_response_paths() { - for (analysis, path) in [ - (json!({"pages":null}), "pages"), - (json!({"pages":[null]}), "pages[0]"), - (json!({"pages":[{"lines":null}]}), "lines"), - (json!({"pages":[{"width":"bad"}]}), "width"), - ] { - let (base, _, server) = mock_server(vec![MockResponse::json(json!({ - "status":"succeeded", - "analyzeResult":analysis - }))]) - .await; - let error = perform_ocr(wire_request( - "azure_ai/doc-intelligence/prebuilt-read", - &base, - json!({}), - )) - .await - .unwrap_err(); - server.await.unwrap(); - assert!(error.to_string().contains(path), "{error}"); - } - } - - #[tokio::test] - async fn rejects_missing_invalid_and_cross_origin_operation_locations() { - for headers in [ - Vec::new(), - vec![("Operation-Location", "/relative".into())], - vec![("Operation-Location", "http://example.com/operation".into())], - vec![( - "Operation-Location", - "http://user:password@127.0.0.1/operation".into(), - )], - ] { - let (base, _, server) = mock_server(vec![MockResponse { - status: 202, - headers, - body: json!({}), - }]) - .await; - let error = perform_ocr(wire_request( - "azure_ai/doc-intelligence/prebuilt-read", - &base, - json!({}), - )) - .await - .unwrap_err(); - server.await.unwrap(); - assert!(error.to_string().contains("operation-location")); - } - } - - #[tokio::test] - async fn polling_deadline_bounds_retry_delay() { - let (base, _, server) = mock_server(vec![ - MockResponse { - status: 202, - headers: vec![("Operation-Location", "{base}/operation".into())], - body: json!({}), - }, - MockResponse { - status: 200, - headers: vec![("Retry-After", "9999".into())], - body: json!({"status":"notStarted"}), - }, - ]) - .await; - let mut request = wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({})); - request.transport.poll_timeout = std::time::Duration::from_millis(100); - - let error = tokio::time::timeout(std::time::Duration::from_secs(1), perform_ocr(request)) - .await - .unwrap() - .unwrap_err(); - server.await.unwrap(); - assert!(error.to_string().contains("timed out")); - } - - #[tokio::test] - async fn model_id_is_encoded_and_dot_segments_are_rejected() { - let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ - "status":"succeeded" - }))]) - .await; - perform_ocr(wire_request( - "azure_ai/doc-intelligence/a ?#é", - &base, - json!({}), - )) - .await - .unwrap(); - server.await.unwrap(); - assert!(seen.lock().unwrap()[0].contains("a%20%3F%23%C3%A9:analyze")); - - for model in [ - "azure_ai/doc-intelligence/.", - "azure_ai/doc-intelligence/..", - ] { - let error = perform_ocr(wire_request(model, "http://127.0.0.1:1", json!({}))) - .await - .unwrap_err(); - assert!(error.to_string().contains("dot segment")); - } - } } diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/mod.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/mod.rs new file mode 100644 index 00000000000..2a5bfe45ff9 --- /dev/null +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/mod.rs @@ -0,0 +1,4 @@ +pub mod cohere_parse_transformation; +pub mod common_utils; +pub mod document_intelligence; +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs new file mode 100644 index 00000000000..2012f740173 --- /dev/null +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs @@ -0,0 +1,330 @@ +use litellm_auth::{InputSource, Sourced}; +use litellm_auth_azure::AzureAuthInputs; +use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl}; +use serde_json::Value; + +use crate::{ + base_llm::ocr::{ + document::{inline_remote_document, validate_inline_document}, + error::Error, + transformation::{ + BaseOcrConfig, LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrRequestContext, + OcrResponseFormat, PreparedOcrRequest, credential_env, + }, + }, + custom_httpx::llm_http_handler::OcrClient, + mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest}, +}; + +const AZURE_AI_OCR_PATH: &str = "/providers/mistral/azure/ocr"; + +const AZURE_AI_API_KEY_ENV: &str = "AZURE_AI_API_KEY"; +const AZURE_AI_API_BASE_ENV: &str = "AZURE_AI_API_BASE"; + +#[derive(Clone, Debug, Default)] +pub struct AzureAiOcrConfig; + +impl BaseOcrConfig for AzureAiOcrConfig { + type OcrParams = OpaqueParams; + type ProviderRequest = MistralOcrRequest; + type Environment = Vec<(String, String)>; + + fn get_supported_ocr_params(&self, model: &str) -> &'static [&'static str] { + MistralOcrConfig.get_supported_ocr_params(model) + } + + fn get_api_key_env_var(&self) -> Option<&'static str> { + Some(AZURE_AI_API_KEY_ENV) + } + + fn map_ocr_params( + &self, + non_default_params: &CallArguments, + model: &str, + ) -> Result { + MistralOcrConfig.map_ocr_params(non_default_params, model) + } + + async fn validate_environment( + &self, + request: &PreparedOcrRequest, + _client: &OcrClient, + ) -> Result { + let config = AzureAuthInputs { + azure_ad_token_provider: request.azure_ad_token_provider.clone(), + ..AzureAuthInputs::from_sourced_optional_params( + &request.optional_params, + &request.input_sources, + )? + }; + self.resolve_headers(&request.connection, &config, &credential_env) + .await + } + + fn get_complete_url( + &self, + request: &PreparedOcrRequest, + _optional_params: &Self::OcrParams, + _environment: &Self::Environment, + ) -> Result { + self.build_ocr_url(request.connection.api_base.as_deref(), &credential_env) + } + + fn transform_ocr_request( + &self, + model: &str, + document: OcrDocument, + optional_params: &OpaqueParams, + headers: &[(String, String)], + ) -> Result { + MistralOcrConfig.transform_ocr_request(model, document, optional_params, headers) + } + + async fn async_transform_ocr_request( + &self, + model: &str, + document: OcrDocument, + optional_params: &OpaqueParams, + headers: &[(String, String)], + context: OcrRequestContext<'_>, + ) -> Result { + let document = inline_remote_document( + context.client.document_fetcher(), + document, + context.connection, + ) + .await?; + self.transform_ocr_request(model, document, optional_params, headers) + } + + fn transform_ocr_response( + &self, + model: &str, + raw_response: &[u8], + request_format: OcrResponseFormat, + ) -> Result { + MistralOcrConfig.transform_ocr_response(model, raw_response, request_format) + } + + fn validate_request_body(&self, body: &Value) -> Result<(), Error> { + validate_inline_document(&crate::custom_httpx::llm_http_handler::body_document(body)?) + } +} + +impl AzureAiOcrConfig { + /// Python `AzureAIOCRConfig.validate_environment` requires the endpoint + /// before it resolves credentials; keep that order so a missing base is + /// reported without invoking any token provider. + pub(super) fn resolve_api_base( + api_base: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + nonblank(api_base.map(str::to_string)) + .or_else(|| nonblank(env_lookup(AZURE_AI_API_BASE_ENV))) + .ok_or(Error::Auth(litellm_auth::Error::MissingApiBase { + provider: "Azure AI", + environment_variable: AZURE_AI_API_BASE_ENV, + })) + } + + async fn resolve_headers( + &self, + connection: &OcrConnection, + config: &AzureAuthInputs, + env_lookup: &(dyn Fn(&str) -> Option + Sync), + ) -> Result, Error> { + Self::resolve_api_base(connection.api_base.as_deref(), env_lookup)?; + if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization") + { + if config.azure_ad_token_provider.is_some() { + super::common_utils::resolve_entra(config, env_lookup).await?; + } + super::common_utils::validate_destination(connection, connection.extra_headers_source)?; + return Ok(connection.extra_headers.clone()); + } + let key = nonblank(connection.api_key.clone()) + .map(|value| Sourced::new(value, connection.api_key_source)) + .or_else(|| { + nonblank(self.get_api_key_env_var().and_then(env_lookup)) + .map(|value| Sourced::new(value, InputSource::Environment)) + }); + if let Some(key) = key { + super::common_utils::validate_destination(connection, key.source())?; + return Ok(bearer_headers(connection, key.value())); + } + let key = super::common_utils::resolve_entra(config, env_lookup) + .await? + .ok_or(Error::MissingAzureAiCredentials)?; + super::common_utils::validate_destination(connection, key.source())?; + Ok(bearer_headers(connection, key.value())) + } + + fn build_ocr_url( + &self, + api_base: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let base = Self::resolve_api_base(api_base, env_lookup)?; + let path: Vec<&str> = AZURE_AI_OCR_PATH.trim_matches('/').split('/').collect(); + ApiUrl::parse(&base) + .and_then(|url| url.complete_path(&path)) + .map(|url| url.into_string()) + .map_err(|_| Error::RequestField { + path: "api_base".into(), + }) + } +} + +fn bearer_headers(connection: &OcrConnection, key: &str) -> Vec<(String, String)> { + std::iter::once(("Authorization".into(), format!("Bearer {key}"))) + .chain(connection.extra_headers.clone()) + .collect() +} + +fn nonblank(value: Option) -> Option { + value + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +#[cfg(test)] +mod tests { + use rstest::{fixture, rstest}; + + use super::*; + + #[fixture] + fn connection() -> OcrConnection { + OcrConnection { + api_key: Some("request-key".into()), + api_base: Some("https://example.com".into()), + ..Default::default() + } + } + + #[rstest] + #[case::base_with_query( + "https://example.com/?tenant=a", + "https://example.com/providers/mistral/azure/ocr?tenant=a" + )] + #[case::complete_endpoint( + "https://example.com/providers/mistral/azure/ocr", + "https://example.com/providers/mistral/azure/ocr" + )] + fn completes_azure_path_and_preserves_query(#[case] api_base: &str, #[case] expected: &str) { + assert_eq!( + AzureAiOcrConfig + .build_ocr_url(Some(api_base), &|_| None) + .unwrap(), + expected + ); + } + + #[test] + fn missing_api_base_is_structured() { + assert!(matches!( + AzureAiOcrConfig::resolve_api_base(None, &|_| None), + Err(Error::Auth(litellm_auth::Error::MissingApiBase { + provider: "Azure AI", + environment_variable: AZURE_AI_API_BASE_ENV, + })) + )); + } + + #[rstest] + #[tokio::test] + async fn supplied_authorization_precedes_keys(connection: OcrConnection) { + let connection = OcrConnection { + extra_headers: vec![("authorization".into(), "Bearer prepared".into())], + ..connection + }; + assert_eq!( + AzureAiOcrConfig + .resolve_headers(&connection, &Default::default(), &|_| { + Some("environment-key".into()) + }) + .await + .unwrap(), + connection.extra_headers + ); + } + + #[rstest] + #[tokio::test] + async fn request_key_precedes_environment_key(connection: OcrConnection) { + assert_eq!( + AzureAiOcrConfig + .resolve_headers(&connection, &Default::default(), &|_| { + Some("environment-key".into()) + }) + .await + .unwrap()[0], + ("Authorization".into(), "Bearer request-key".into()) + ); + } + + #[tokio::test] + async fn request_endpoint_cannot_receive_environment_key() { + let connection = OcrConnection { + api_base: Some("https://request.example".into()), + api_base_source: InputSource::Request, + ..Default::default() + }; + + let error = AzureAiOcrConfig + .resolve_headers(&connection, &Default::default(), &|name| { + (name == AZURE_AI_API_KEY_ENV).then(|| "environment-key".into()) + }) + .await + .unwrap_err(); + + assert!( + error + .to_string() + .contains("request-controlled Azure endpoint") + ); + } + + #[tokio::test] + async fn request_endpoint_accepts_request_owned_key() { + let connection = OcrConnection { + api_key: Some("request-key".into()), + api_key_source: InputSource::Request, + api_base: Some("https://request.example".into()), + api_base_source: InputSource::Request, + ..Default::default() + }; + + let headers = AzureAiOcrConfig + .resolve_headers(&connection, &Default::default(), &|_| None) + .await + .unwrap(); + + assert_eq!( + headers[0], + ("Authorization".into(), "Bearer request-key".into()) + ); + } + + #[tokio::test] + async fn environment_supplies_api_base_and_bearer_key() { + let env = |name: &str| match name { + AZURE_AI_API_BASE_ENV => Some("https://env.example".to_string()), + AZURE_AI_API_KEY_ENV => Some("env-key".to_string()), + _ => None, + }; + let connection = OcrConnection::default(); + + let headers = AzureAiOcrConfig + .resolve_headers(&connection, &Default::default(), &env) + .await + .unwrap(); + let url = AzureAiOcrConfig.build_ocr_url(None, &env).unwrap(); + + assert_eq!( + headers, + [("Authorization".to_string(), "Bearer env-key".to_string())] + ); + assert_eq!(url, "https://env.example/providers/mistral/azure/ocr"); + } +} diff --git a/litellm-rust/crates/providers/src/base_llm/anthropic_messages/mod.rs b/litellm-rust/crates/llms/src/base_llm/anthropic_messages/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/anthropic_messages/mod.rs rename to litellm-rust/crates/llms/src/base_llm/anthropic_messages/mod.rs diff --git a/litellm-rust/crates/providers/src/base_llm/anthropic_messages/transformation.rs b/litellm-rust/crates/llms/src/base_llm/anthropic_messages/transformation.rs similarity index 88% rename from litellm-rust/crates/providers/src/base_llm/anthropic_messages/transformation.rs rename to litellm-rust/crates/llms/src/base_llm/anthropic_messages/transformation.rs index 37bf8884ec0..5b4afb601d2 100644 --- a/litellm-rust/crates/providers/src/base_llm/anthropic_messages/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/anthropic_messages/transformation.rs @@ -1,5 +1,8 @@ -use crate::messages::Error; -use crate::messages::types::{AnthropicMessagesRequest, AnthropicMessagesResponse}; +use litellm_types::llms::anthropic_messages::{ + anthropic_request::AnthropicMessagesRequest, anthropic_response::AnthropicMessagesResponse, +}; + +use crate::base_llm::chat::transformation::Error; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MessagesAuthStrategy { diff --git a/litellm-rust/crates/providers/src/base_llm/audio_transcription/mod.rs b/litellm-rust/crates/llms/src/base_llm/audio_transcription/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/audio_transcription/mod.rs rename to litellm-rust/crates/llms/src/base_llm/audio_transcription/mod.rs diff --git a/litellm-rust/crates/providers/src/base_llm/audio_transcription/transformation.rs b/litellm-rust/crates/llms/src/base_llm/audio_transcription/transformation.rs similarity index 74% rename from litellm-rust/crates/providers/src/base_llm/audio_transcription/transformation.rs rename to litellm-rust/crates/llms/src/base_llm/audio_transcription/transformation.rs index b478bd4caab..dd4588732be 100644 --- a/litellm-rust/crates/providers/src/base_llm/audio_transcription/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/audio_transcription/transformation.rs @@ -1,9 +1,25 @@ +use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use crate::audio_transcription::Error; -use crate::audio_transcription::types::{ - AudioTranscriptionRequestData, AudioTranscriptionResponseData, -}; +use crate::base_llm::chat::transformation::Error; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct AudioTranscriptionRequestData { + pub body: Value, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct AudioTranscriptionResponseData { + pub text: String, +} + +impl AudioTranscriptionResponseData { + pub fn into_json(self) -> Value { + serde_json::json!({ + "text": self.text, + }) + } +} #[derive(Clone, Debug, PartialEq, Eq)] pub enum AudioTranscriptionAuth { diff --git a/litellm-rust/crates/core/src/chat_completions/streaming.rs b/litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs similarity index 100% rename from litellm-rust/crates/core/src/chat_completions/streaming.rs rename to litellm-rust/crates/llms/src/base_llm/base_model_iterator.rs diff --git a/litellm-rust/crates/providers/src/base_llm/chat/mod.rs b/litellm-rust/crates/llms/src/base_llm/chat/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/base_llm/chat/mod.rs rename to litellm-rust/crates/llms/src/base_llm/chat/mod.rs diff --git a/litellm-rust/crates/providers/src/base_llm/chat/transformation.rs b/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs similarity index 82% rename from litellm-rust/crates/providers/src/base_llm/chat/transformation.rs rename to litellm-rust/crates/llms/src/base_llm/chat/transformation.rs index 5d81dc1a85e..ac0450c25f0 100644 --- a/litellm-rust/crates/providers/src/base_llm/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs @@ -1,10 +1,39 @@ +use litellm_types::{ + llms::openai::{ChatMessage, ChatMessageContent}, + utils::ChatCompletionsResponse, +}; use serde_json::{Map, Value}; -use crate::chat::Error; -use crate::chat::types::{ - ChatCompletionsResponse, ChatMessage, ChatMessageContent, ProviderChatRequestData, - ProviderChatResponseData, -}; +#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("expected {expected}, got {actual}")] + InvalidType { + expected: &'static str, + actual: &'static str, + }, + #[error("missing required field: {0}")] + MissingField(&'static str), + #[error("invalid request: {0}")] + InvalidRequest(String), + #[error("invalid response: {0}")] + InvalidResponse(String), + #[error("unsupported: {0}")] + Unsupported(&'static str), + #[error(transparent)] + Auth(#[from] litellm_auth::Error), +} + +/// The provider-shaped request body a config produces. Named rather than a bare +/// `Value` so the transform contract stays a typed one, mirroring +/// [`crate::base_llm::audio_transcription::transformation::AudioTranscriptionRequestData`]. +pub struct ProviderChatRequestData { + pub body: Value, +} + +/// The raw provider response body handed back to a config for normalization. +pub struct ProviderChatResponseData { + pub body: Value, +} pub const STREAM_PARAM: &str = "stream"; diff --git a/litellm-rust/crates/providers/src/base_llm/mod.rs b/litellm-rust/crates/llms/src/base_llm/mod.rs similarity index 53% rename from litellm-rust/crates/providers/src/base_llm/mod.rs rename to litellm-rust/crates/llms/src/base_llm/mod.rs index b7a1f696440..8ed37da4573 100644 --- a/litellm-rust/crates/providers/src/base_llm/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/mod.rs @@ -1,3 +1,6 @@ pub mod anthropic_messages; pub mod audio_transcription; +pub mod base_model_iterator; pub mod chat; +pub mod ocr; +pub mod responses; diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/document.rs b/litellm-rust/crates/llms/src/base_llm/ocr/document.rs new file mode 100644 index 00000000000..8737232a075 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/ocr/document.rs @@ -0,0 +1,225 @@ +use base64::{Engine, engine::general_purpose::STANDARD}; +use data_url::{DataUrl, DataUrlError, forgiving_base64::DecodeError, mime::Mime}; +use reqwest::Url; + +use crate::{ + base_llm::ocr::{ + error::Error, + transformation::{ + OCR_INLINE_MAX_BYTES, OCR_MAX_FETCH_REDIRECTS, OcrConnection, OcrDocument, + }, + }, + custom_httpx::{ + media::{DownloadPolicy, Error as MediaError, MediaFetcher}, + transport::Error as TransportError, + }, +}; + +pub struct InlineDocument<'a>(DataUrl<'a>); + +impl<'a> InlineDocument<'a> { + pub fn parse(source: &'a str) -> Result, Error> { + match DataUrl::process(source) { + Ok(url) => Ok(Some(Self(url))), + Err(DataUrlError::NotADataUrl) => Ok(None), + Err(DataUrlError::NoComma) => Err(Error::InvalidDataUri), + } + } + + pub fn mime_type(&self) -> &Mime { + self.0.mime_type() + } + + pub fn decode(&self, max_bytes: usize) -> Result, Error> { + let mut body = Vec::new(); + self.0 + .decode(|bytes| { + if bytes.len() > max_bytes.saturating_sub(body.len()) { + return Err(Error::InlineDocumentTooLarge); + } + body.extend_from_slice(bytes); + Ok(()) + }) + .map_err(|error| match error { + DecodeError::InvalidBase64(_) => Error::InvalidDataUri, + DecodeError::WriteError(error) => error, + })?; + Ok(body) + } +} + +pub fn validate_inline_document(document: &OcrDocument) -> Result<(), Error> { + let inline = InlineDocument::parse(document.source())?.ok_or(Error::InvalidDataUri)?; + inline.decode(OCR_INLINE_MAX_BYTES)?; + Ok(()) +} + +pub async fn inline_remote_document( + fetcher: &MediaFetcher, + document: OcrDocument, + connection: &OcrConnection, +) -> Result { + let source = document.source(); + if !document.is_remote() { + validate_inline_document(&document)?; + return Ok(document); + } + let url = Url::parse(source).map_err(|_| Error::RequestField { + path: "document URL".into(), + })?; + let downloaded = fetcher + .fetch( + url, + DownloadPolicy { + timeout: connection.timeout, + max_bytes: connection.max_download_bytes, + max_redirects: OCR_MAX_FETCH_REDIRECTS, + }, + ) + .await + .map_err(map_media_error)?; + let result = document.with_source(format!( + "data:{};base64,{}", + downloaded.content_type, + STANDARD.encode(downloaded.bytes) + )); + validate_inline_document(&result)?; + Ok(result) +} + +fn map_media_error(error: MediaError) -> Error { + match error { + MediaError::BlockedUrl => Error::BlockedDocumentUrl, + MediaError::DownloadDisabled => Error::DownloadDisabled, + MediaError::DownloadTooLarge => Error::DownloadTooLarge, + MediaError::TooManyRedirects => Error::TooManyRedirects, + MediaError::MissingRedirectLocation => Error::MissingRedirectLocation, + MediaError::InvalidRedirect => Error::InvalidRedirect, + MediaError::Http(status) => TransportError::Http { + status, + body: "OCR document download failed".into(), + } + .into(), + MediaError::Timeout => TransportError::Http { + status: 408, + body: "OCR document download timed out".into(), + } + .into(), + MediaError::Transport(error) => error.into(), + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap as Map; + + use super::*; + + fn document(source: &str) -> OcrDocument { + OcrDocument::DocumentUrl { + document_url: source.into(), + extra_fields: Map::new(), + } + } + + #[test] + fn decodes_data_urls_and_limits_decoded_size() { + for (source, expected) in [ + ("data:application/pdf;base64,YWJj", b"abc".as_slice()), + ("DATA:application/pdf;BASE64,YWI", b"ab".as_slice()), + ("data:,a%20b%00%FF", b"a b\0\xff".as_slice()), + ] { + let inline = InlineDocument::parse(source).unwrap().unwrap(); + assert_eq!(inline.decode(expected.len()).unwrap(), expected); + assert!(matches!( + inline.decode(expected.len() - 1), + Err(Error::InlineDocumentTooLarge) + )); + } + } + + #[test] + fn preserves_mime_parameters_and_standard_default() { + let inline = InlineDocument::parse("data:application/pdf;version=1.7;base64,YQ==") + .unwrap() + .unwrap(); + assert!(inline.mime_type().matches("application", "pdf")); + assert_eq!(inline.mime_type().get_parameter("version"), Some("1.7")); + let default = InlineDocument::parse("data:,a").unwrap().unwrap(); + assert!(default.mime_type().matches("text", "plain")); + assert_eq!( + default.mime_type().get_parameter("charset"), + Some("US-ASCII") + ); + } + + #[test] + fn rejects_invalid_inline_documents() { + for source in [ + "https://example.com/document.pdf", + "data:application/pdf;base64", + "data:application/pdf;base64,INVALID!", + ] { + assert!(validate_inline_document(&document(source)).is_err()); + } + } + + #[tokio::test] + async fn remote_conversion_preserves_kind_and_isolates_provider_credentials() { + use tokio::{ + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + }; + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.unwrap(); + let mut request = vec![0_u8; 2048]; + let count = socket.read(&mut request).await.unwrap(); + socket + .write_all(b"HTTP/1.1 200 OK\r\nContent-Type: image/png; charset=binary\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabc") + .await + .unwrap(); + String::from_utf8_lossy(&request[..count]).into_owned() + }); + let mut provider_headers = reqwest::header::HeaderMap::new(); + provider_headers.insert( + reqwest::header::AUTHORIZATION, + reqwest::header::HeaderValue::from_static("Bearer provider-secret"), + ); + let provider_http = reqwest::Client::builder() + .default_headers(provider_headers) + .build() + .unwrap(); + let document_http = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .build() + .unwrap(); + let client = crate::custom_httpx::llm_http_handler::OcrClient::for_test( + provider_http, + document_http, + ); + let converted = inline_remote_document( + client.document_fetcher(), + OcrDocument::ImageUrl { + image_url: format!("http://{address}/image"), + extra_fields: Map::from_iter([("detail".into(), Some("high".into()))]), + }, + &OcrConnection::default(), + ) + .await + .unwrap(); + let request = server.await.unwrap(); + + assert_eq!( + converted, + OcrDocument::ImageUrl { + image_url: "data:image/png;base64,YWJj".into(), + extra_fields: Map::from_iter([("detail".into(), Some("high".into()))]), + } + ); + assert!(!request.to_ascii_lowercase().contains("authorization")); + assert!(!request.contains("provider-secret")); + } +} diff --git a/litellm-rust/crates/core/src/ocr/error.rs b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs similarity index 92% rename from litellm-rust/crates/core/src/ocr/error.rs rename to litellm-rust/crates/llms/src/base_llm/ocr/error.rs index 4906b5515b9..c3f481d7d44 100644 --- a/litellm-rust/crates/core/src/ocr/error.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs @@ -95,15 +95,15 @@ pub enum Error { #[error(transparent)] Auth(#[from] litellm_auth::Error), #[error(transparent)] - Transport(#[from] crate::transport::Error), + Transport(#[from] crate::custom_httpx::transport::Error), #[error(transparent)] - Params(#[from] crate::params::Error), + Params(#[from] litellm_core_utils::params::Error), #[error(transparent)] - Headers(#[from] crate::http_utils::HeaderError), + Headers(#[from] crate::custom_httpx::http_handler::HeaderError), } -impl From for Error { - fn from(error: crate::call_arguments::ArgumentError) -> Self { +impl From for Error { + fn from(error: litellm_core_utils::call_arguments::ArgumentError) -> Self { Self::RequestField { path: format!("optional_params.{}", error.path), } @@ -114,7 +114,9 @@ impl Error { pub fn http_status_code(&self) -> Option { match self { Self::Provider { status, .. } - | Self::Transport(crate::transport::Error::Http { status, .. }) => Some(*status), + | Self::Transport(crate::custom_httpx::transport::Error::Http { status, .. }) => { + Some(*status) + } error if error.is_request() => Some(400), _ => None, } diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/mod.rs b/litellm-rust/crates/llms/src/base_llm/ocr/mod.rs new file mode 100644 index 00000000000..7194efbb203 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/ocr/mod.rs @@ -0,0 +1,3 @@ +pub mod document; +pub mod error; +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs new file mode 100644 index 00000000000..e6fe5d9556d --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs @@ -0,0 +1,667 @@ +use std::{collections::BTreeMap, future::Future, time::Duration}; + +use litellm_auth::{InputSource, Sourced, TokenProviderHandle}; +use litellm_core_utils::{ + call_arguments::CallArguments, + serde_compat::{FiniteF64, LaxI64}, +}; +use serde::{ + Deserialize, Serialize, + de::{DeserializeOwned, IntoDeserializer}, +}; +use serde_json::{Map, Value}; +use serde_with::serde_as; + +use crate::{ + base_llm::ocr::error::Error, + custom_httpx::llm_http_handler::{ + CallHooks, OcrClient, read_response_bytes, transform_request_body, + }, +}; + +pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024; +pub const OCR_HTTP_TIMEOUT_SECS: u64 = 600; +pub const OCR_CONNECT_TIMEOUT_SECS: u64 = 10; +pub const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024; +pub const OCR_DOWNLOAD_MAX_BYTES: u64 = 50 * 1024 * 1024; +pub const OCR_MAX_FETCH_REDIRECTS: usize = 10; +pub const OCR_POLL_TIMEOUT_SECS: u64 = 120; +pub const OCR_POLL_RETRY_SECS: u64 = 2; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum OcrDocument { + #[serde(rename = "document_url")] + DocumentUrl { + document_url: String, + #[serde(flatten)] + extra_fields: BTreeMap>, + }, + #[serde(rename = "image_url")] + ImageUrl { + image_url: String, + #[serde(flatten)] + extra_fields: BTreeMap>, + }, +} + +impl OcrDocument { + pub fn source(&self) -> &str { + match self { + Self::DocumentUrl { document_url, .. } => document_url, + Self::ImageUrl { image_url, .. } => image_url, + } + } + + pub fn is_remote(&self) -> bool { + let source = self.source(); + source.starts_with("http://") || source.starts_with("https://") + } + + pub fn with_source(self, source: String) -> Self { + match self { + Self::DocumentUrl { extra_fields, .. } => Self::DocumentUrl { + document_url: source, + extra_fields, + }, + Self::ImageUrl { extra_fields, .. } => Self::ImageUrl { + image_url: source, + extra_fields, + }, + } + } +} + +impl TryFrom for OcrDocument { + type Error = Error; + + fn try_from(value: Value) -> Result { + decode_request_value(value, "document") + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum OcrResponseFormat { + #[default] + Litellm, + Native, +} + +#[derive(Clone, Default)] +pub struct OcrCredentialInputs { + pub api_key: Option>, + pub dynamic_api_key: Option>, + pub api_base: Option>, + pub dynamic_api_base: Option>, +} + +impl OcrCredentialInputs { + pub fn new( + api_key: Option, + api_key_source: InputSource, + api_base: Option, + api_base_source: InputSource, + ) -> Self { + Self { + api_key: nonblank(api_key).map(|value| Sourced::new(value, api_key_source)), + dynamic_api_key: None, + api_base: nonblank(api_base).map(|value| Sourced::new(value, api_base_source)), + dynamic_api_base: None, + } + } +} + +#[derive(Clone)] +pub struct OcrTransportConfig { + pub extra_headers: Vec<(String, String)>, + pub extra_headers_source: InputSource, + pub timeout: Duration, + pub max_download_bytes: u64, + pub max_response_bytes: usize, + pub poll_timeout: Duration, +} + +impl Default for OcrTransportConfig { + fn default() -> Self { + Self { + extra_headers: Vec::new(), + extra_headers_source: InputSource::Deployment, + timeout: Duration::from_secs(OCR_HTTP_TIMEOUT_SECS), + max_download_bytes: OCR_DOWNLOAD_MAX_BYTES, + max_response_bytes: OCR_RESPONSE_MAX_BYTES, + poll_timeout: Duration::from_secs(OCR_POLL_TIMEOUT_SECS), + } + } +} + +impl OcrTransportConfig { + pub fn with_overrides( + self, + extra_headers: Vec<(String, String)>, + extra_headers_source: InputSource, + timeout: Option, + ) -> Self { + Self { + extra_headers, + extra_headers_source, + timeout: timeout.unwrap_or(self.timeout), + ..self + } + } +} + +fn nonblank(value: Option) -> Option { + value + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +#[derive(Clone)] +pub struct OcrConnection { + pub api_key: Option, + pub api_key_source: InputSource, + pub api_base: Option, + pub api_base_source: InputSource, + pub extra_headers: Vec<(String, String)>, + pub extra_headers_source: InputSource, + pub timeout: Duration, + pub max_download_bytes: u64, + pub max_response_bytes: usize, + pub poll_timeout: Duration, +} + +impl OcrConnection { + pub fn new(credentials: ResolvedOcrCredentials, transport: OcrTransportConfig) -> Self { + let api_key_source = credentials + .api_key + .as_ref() + .map(Sourced::source) + .unwrap_or(InputSource::Deployment); + let api_base_source = credentials + .api_base + .as_ref() + .map(Sourced::source) + .unwrap_or(InputSource::Deployment); + Self { + api_key: credentials.api_key.map(Sourced::into_value), + api_key_source, + api_base: credentials.api_base.map(Sourced::into_value), + api_base_source, + extra_headers: transport.extra_headers, + extra_headers_source: transport.extra_headers_source, + timeout: transport.timeout, + max_download_bytes: transport.max_download_bytes, + max_response_bytes: transport.max_response_bytes, + poll_timeout: transport.poll_timeout, + } + } +} + +impl Default for OcrConnection { + fn default() -> Self { + Self::new( + ResolvedOcrCredentials::default(), + OcrTransportConfig::default(), + ) + } +} + +#[derive(Clone, Default)] +pub struct ResolvedOcrCredentials { + pub api_key: Option>, + pub api_base: Option>, +} + +pub struct PreparedOcrRequest { + pub model: String, + pub document: OcrDocument, + pub connection: OcrConnection, + /// Whether the caller handed over the document as is, so the wire body's document + /// is the caller's own input rather than something the route prepared. + pub caller_document: bool, + pub optional_params: CallArguments, + pub input_sources: BTreeMap, + pub azure_ad_token_provider: Option, +} + +impl PreparedOcrRequest { + pub fn response_format(&self) -> Result { + response_format(&self.optional_params) + } +} + +pub fn response_format(optional_params: &CallArguments) -> Result { + optional_params + .get("req_format") + .filter(|value| !value.is_null()) + .map(|value| serde_json::from_value(value.clone()).map_err(|_| Error::RequestFormat)) + .transpose() + .map(|format| format.unwrap_or_default()) +} + +#[serde_as] +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct OcrPageDimensions { + #[serde_as(deserialize_as = "Option")] + pub dpi: Option, + #[serde_as(deserialize_as = "Option")] + pub height: Option, + #[serde_as(deserialize_as = "Option")] + pub width: Option, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct OcrPageImage { + pub image_base64: Option, + pub bbox: Option>, + #[serde(flatten)] + pub extra_fields: Map, +} + +#[serde_as] +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct OcrPage { + #[serde_as(deserialize_as = "LaxI64")] + pub index: i64, + pub markdown: String, + pub images: Option>, + pub dimensions: Option, + #[serde(flatten)] + pub extra_fields: Map, +} + +#[serde_as] +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct OcrUsageInfo { + #[serde_as(deserialize_as = "Option")] + pub pages_processed: Option, + #[serde_as(deserialize_as = "Option")] + pub pages_processed_annotation: Option, + #[serde_as(deserialize_as = "Option")] + pub credits: Option, + #[serde_as(deserialize_as = "Option")] + pub doc_size_bytes: Option, + #[serde(flatten)] + pub extra_fields: Map, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct LiteLLMOcrResponse { + pub pages: Vec, + pub model: String, + pub document_annotation: Option, + pub usage_info: Option, + pub content: Option, + pub tables: Option>>, + #[serde(rename = "keyValuePairs")] + pub key_value_pairs: Option>>, + #[serde(default = "ocr_object")] + pub object: String, + #[serde(flatten)] + pub extra_fields: Map, + #[serde(skip_serializing_if = "Option::is_none")] + pub provider_native_response: Option>, +} + +impl LiteLLMOcrResponse { + pub fn new(model: impl Into, pages: Vec) -> Self { + Self { + pages, + model: model.into(), + document_annotation: None, + usage_info: None, + content: None, + tables: None, + key_value_pairs: None, + object: ocr_object(), + extra_fields: Map::new(), + provider_native_response: None, + } + } + + pub fn into_json(self) -> Value { + serde_json::to_value(self).expect("OCR response fields are JSON-compatible") + } +} + +fn ocr_object() -> String { + "ocr".into() +} + +#[derive(Debug)] +pub struct DecodedOcrResponse { + pub data: T, + pub native: Option>, + pub text: String, +} + +pub fn decode_request_value(value: Value, prefix: &str) -> Result { + serde_path_to_error::deserialize(value.into_deserializer()).map_err(|error| { + Error::RequestField { + path: format!("{prefix}.{}", error.path()), + } + }) +} + +pub fn decode_response_value(value: Value, prefix: &str) -> Result { + serde_path_to_error::deserialize(value.into_deserializer()).map_err(|error| { + Error::ResponseField { + path: format!("{prefix}.{}", error.path()), + } + }) +} + +pub fn decode_response( + bytes: &[u8], + native: bool, +) -> Result, Error> { + let mut deserializer = serde_json::Deserializer::from_slice(bytes); + let data = serde_path_to_error::deserialize(&mut deserializer).map_err(|error| { + Error::ResponseField { + path: error.path().to_string(), + } + })?; + deserializer.end().map_err(|_| Error::ResponseField { + path: "response".into(), + })?; + let native = if native { + Some( + serde_json::from_slice(bytes).map_err(|_| Error::ResponseField { + path: "response".into(), + })?, + ) + } else { + None + }; + Ok(DecodedOcrResponse { + data, + native, + text: String::from_utf8_lossy(bytes).into_owned(), + }) +} + +const HEALTH_CHECK_PDF_DATA_URI: &str = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="; + +/// Output of `validate_environment`: whatever a provider resolves up front +/// (headers at minimum; Vertex also carries the project id). +pub trait OcrEnvironment: Send + Sync { + fn headers(&self) -> &[(String, String)]; +} + +impl OcrEnvironment for Vec<(String, String)> { + fn headers(&self) -> &[(String, String)] { + self + } +} + +#[derive(Clone, Copy)] +pub struct OcrRequestContext<'a> { + pub client: &'a OcrClient, + pub connection: &'a OcrConnection, +} + +#[derive(Clone, Copy)] +pub struct OcrResponseContext<'a> { + pub client: &'a OcrClient, + pub connection: &'a OcrConnection, + pub hooks: &'a dyn CallHooks, + pub request_format: OcrResponseFormat, + pub url: &'a str, + pub headers: &'a [(String, String)], +} + +pub trait BaseOcrConfig: Send + Sync + Sized + 'static { + type OcrParams: Send + Sync; + type ProviderRequest: Serialize + Send; + type Environment: OcrEnvironment; + + fn get_supported_ocr_params(&self, _model: &str) -> &'static [&'static str] { + &[] + } + + fn get_api_key_env_var(&self) -> Option<&'static str> { + None + } + + fn resolve_connection_params(&self, inputs: OcrCredentialInputs) -> ResolvedOcrCredentials { + ResolvedOcrCredentials { + api_key: inputs + .dynamic_api_key + .filter(|value| !value.value().is_empty()) + .or(inputs.api_key), + api_base: inputs + .dynamic_api_base + .filter(|value| !value.value().is_empty()) + .or(inputs.api_base), + } + } + + fn get_health_check_document(&self) -> OcrDocument { + OcrDocument::DocumentUrl { + document_url: HEALTH_CHECK_PDF_DATA_URI.into(), + extra_fields: Default::default(), + } + } + + fn map_ocr_params( + &self, + non_default_params: &CallArguments, + model: &str, + ) -> Result; + + fn validate_environment( + &self, + request: &PreparedOcrRequest, + client: &OcrClient, + ) -> impl Future> + Send; + + fn get_complete_url( + &self, + request: &PreparedOcrRequest, + optional_params: &Self::OcrParams, + environment: &Self::Environment, + ) -> Result; + + fn transform_ocr_request( + &self, + model: &str, + document: OcrDocument, + optional_params: &Self::OcrParams, + headers: &[(String, String)], + ) -> Result; + + fn async_transform_ocr_request( + &self, + model: &str, + document: OcrDocument, + optional_params: &Self::OcrParams, + headers: &[(String, String)], + _context: OcrRequestContext<'_>, + ) -> impl Future> + Send { + async move { self.transform_ocr_request(model, document, optional_params, headers) } + } + + fn transform_ocr_response( + &self, + model: &str, + raw_response: &[u8], + request_format: OcrResponseFormat, + ) -> Result; + + fn async_transform_ocr_response( + &self, + model: &str, + raw_response: reqwest::Response, + context: OcrResponseContext<'_>, + ) -> impl Future> + Send { + async move { + let bytes = + read_response_bytes(raw_response, context.connection.max_response_bytes).await?; + context.hooks.response_received(&bytes).await?; + self.transform_ocr_response(model, &bytes, context.request_format) + } + } + + fn get_error_class( + &self, + error_message: String, + status_code: u16, + headers: Vec<(String, String)>, + ) -> Error { + Error::Provider { + status: status_code, + body: error_message, + headers, + } + } + + /// Provider-specific check applied to the composed body, both before and + /// after guardrail hooks. Defaults to accepting any body. + fn validate_request_body(&self, _body: &Value) -> Result<(), Error> { + Ok(()) + } + + /// Rust counterpart of `BaseLLMHTTPHandler._async_prepare_ocr_request`: + /// map params, validate environment, build URL, transform, compose body. + fn prepare_request( + &self, + request: &PreparedOcrRequest, + client: &OcrClient, + hooks: &dyn CallHooks, + ) -> impl Future> + Send { + async move { + let params = self.map_ocr_params(&request.optional_params, &request.model)?; + let environment = self.validate_environment(request, client).await?; + let url = self.get_complete_url(request, ¶ms, &environment)?; + let headers = environment.headers(); + let body = self + .async_transform_ocr_request( + &request.model, + request.document.clone(), + ¶ms, + headers, + OcrRequestContext { + client, + connection: &request.connection, + }, + ) + .await?; + transform_request_body(self, client, request, &url, headers, body, hooks).await + } + } +} + +pub fn decode_and_normalize_response( + model: &str, + raw_response: &[u8], + request_format: OcrResponseFormat, + normalize: impl FnOnce(&str, T) -> Result, +) -> Result { + let decoded = decode_response(raw_response, request_format == OcrResponseFormat::Native)?; + Ok(LiteLLMOcrResponse { + provider_native_response: decoded.native, + ..normalize(model, decoded.data)? + }) +} + +pub fn credential_env(name: &str) -> Option { + std::env::var(name).ok() +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn normalized_response_rejects_invalid_shared_fields() { + for fields in [ + json!({"pages":[{}]}), + json!({"pages":[{"index":0,"markdown":false}]}), + json!({"pages":[{"index":0,"markdown":"","images":[{"bbox":[]}]}]}), + json!({"usage_info":{"pages_processed":1.5}}), + json!({"tables":[false]}), + json!({"keyValuePairs":[[]]}), + json!({"provider_native_response":[]}), + ] { + let payload: Map = json!({"model":"model", "pages":[]}) + .as_object() + .unwrap() + .iter() + .chain(fields.as_object().unwrap()) + .map(|(key, value)| (key.clone(), value.clone())) + .collect(); + assert!(serde_json::from_value::(Value::Object(payload)).is_err()); + } + assert!( + serde_json::from_value::(json!({ + "type":"image_url", "image_url":"https://example.com/image", "detail":42 + })) + .is_err() + ); + } + + #[test] + fn numeric_coercion_preserves_integer_precision_and_rejects_fractional_values() { + for (value, expected) in [ + (json!("9007199254740993.0"), 9_007_199_254_740_993), + (json!("+2.000"), 2), + (json!("1_000"), 1000), + (json!(true), 1), + (json!(2.0), 2), + ] { + let page: OcrPage = + serde_json::from_value(json!({"index":value,"markdown":""})).unwrap(); + assert_eq!(page.index, expected); + } + for value in [ + json!("1e2"), + json!(".0"), + json!("2."), + json!("_2"), + json!("2__0"), + json!(2.5), + json!(null), + ] { + assert!( + serde_json::from_value::(json!({"index":value,"markdown":""})).is_err() + ); + } + } + + #[rstest::rstest] + #[case::document_url("document_url", "document_name", "application/pdf")] + #[case::image_url("image_url", "detail", "image/png")] + fn document_variants_preserve_provider_fields_when_rewriting_sources( + #[case] kind: &str, + #[case] field: &str, + #[case] mime_type: &str, + #[values(json!("kept"), Value::Null)] extra: Value, + ) { + let original = "https://example.com/input"; + let replacement = format!("data:{mime_type};base64,AA=="); + let document: OcrDocument = + serde_json::from_value(json!({"type": kind, kind: original, field: extra})).unwrap(); + assert_eq!(document.source(), original); + assert_eq!( + serde_json::to_value(document.with_source(replacement.clone())).unwrap(), + json!({"type": kind, kind: replacement, field: extra}) + ); + } + + #[test] + fn response_serialization_flattens_extra_fields_and_omits_absent_native_response() { + let response = LiteLLMOcrResponse { + extra_fields: json!({"provider_field":"kept"}) + .as_object() + .unwrap() + .clone(), + ..LiteLLMOcrResponse::new("model", vec![]) + }; + let serialized = response.into_json(); + assert_eq!(serialized["provider_field"], "kept"); + assert!(serialized.get("provider_native_response").is_none()); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/responses/mod.rs b/litellm-rust/crates/llms/src/base_llm/responses/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/responses/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs b/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs new file mode 100644 index 00000000000..0d9cfcfd4cd --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/responses/transformation.rs @@ -0,0 +1,180 @@ +use litellm_types::responses::streaming_websocket::{ResponsesWsEvent, ResponsesWsTransformResult}; + +use crate::base_llm::chat::transformation::Error; + +pub const OPENAI_RESPONSES_DEFAULT_API_BASE: &str = "https://api.openai.com/v1"; +pub const OPENAI_RESPONSES_PATH: &str = "/responses"; + +pub trait ResponsesWebSocketProviderConfig: Sync { + fn supports_native_websocket(&self) -> bool { + false + } + + fn model_in_websocket_url(&self) -> bool { + true + } + + fn complete_websocket_url(&self, api_base: Option<&str>, model: &str) -> String { + complete_websocket_url(api_base, model, self.model_in_websocket_url()) + } + + fn transform_ws_request( + &self, + event: &ResponsesWsEvent, + model: &str, + ) -> Result; + + fn transform_ws_response( + &self, + event: &ResponsesWsEvent, + model: &str, + ) -> Result; +} + +pub fn complete_websocket_url( + api_base: Option<&str>, + model: &str, + model_in_websocket_url: bool, +) -> String { + let base = api_base + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or(OPENAI_RESPONSES_DEFAULT_API_BASE); + let (base_without_query, query) = base + .split_once('?') + .map_or((base, None), |(value, query)| (value, Some(query))); + let response_url = format!( + "{}{}", + base_without_query.trim_end_matches('/'), + OPENAI_RESPONSES_PATH + ); + let scheme_flipped = if let Some(rest) = response_url.strip_prefix("https://") { + format!("wss://{rest}") + } else if let Some(rest) = response_url.strip_prefix("http://") { + format!("ws://{rest}") + } else { + response_url + }; + let url = query.map_or(scheme_flipped.clone(), |value| { + format!("{scheme_flipped}?{value}") + }); + if !model_in_websocket_url + || query.is_some_and(|value| { + value + .split('&') + .any(|part| part.split('=').next() == Some("model")) + }) + { + return url; + } + format!( + "{url}{}model={}", + if query.is_some() { "&" } else { "?" }, + percent_encode(model) + ) +} + +fn percent_encode(value: &str) -> String { + value + .bytes() + .map(|byte| { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'.' | b'_' | b'~') { + format!("{}", byte as char) + } else { + format!("%{byte:02X}") + } + }) + .collect() +} + +pub fn enforce_model(event: &ResponsesWsEvent, model: &str) -> ResponsesWsEvent { + if !event.is_response_create() { + return event.clone(); + } + let mut enforced = event.clone(); + let has_flat_model = enforced.data.contains_key("model"); + if let Some(response) = enforced + .data + .get_mut("response") + .and_then(serde_json::Value::as_object_mut) + { + response.insert( + "model".to_string(), + serde_json::Value::String(model.to_string()), + ); + if has_flat_model { + enforced.data.insert( + "model".to_string(), + serde_json::Value::String(model.to_string()), + ); + } + } else { + enforced.data.insert( + "model".to_string(), + serde_json::Value::String(model.to_string()), + ); + } + enforced +} + +#[cfg(test)] +mod tests { + use super::*; + + fn event(value: serde_json::Value) -> ResponsesWsEvent { + serde_json::from_value(value).expect("valid event") + } + + #[test] + fn url_construction_matches_python_defaults_and_query_behavior() { + assert_eq!( + complete_websocket_url(None, "gpt-5", true), + "wss://api.openai.com/v1/responses?model=gpt-5" + ); + assert_eq!( + complete_websocket_url(Some("http://localhost:8080/"), "gpt 5", true), + "ws://localhost:8080/responses?model=gpt%205" + ); + assert_eq!( + complete_websocket_url(Some("https://example.test/v1?foo=bar"), "gpt-5", true), + "wss://example.test/v1/responses?foo=bar&model=gpt-5" + ); + assert_eq!( + complete_websocket_url(Some("https://example.test?model=existing"), "gpt-5", true), + "wss://example.test/responses?model=existing" + ); + } + + #[test] + fn enforce_model_overrides_flat_and_nested_values() { + let flat = enforce_model( + &event(serde_json::json!({"type":"response.create","model":"wrong"})), + "gpt-5", + ); + assert_eq!(flat.model(), Some("gpt-5")); + let nested = enforce_model( + &event(serde_json::json!({ + "type":"response.create", + "model":"wrong", + "response":{"model":"also-wrong"} + })), + "gpt-5", + ); + assert_eq!(nested.model(), Some("gpt-5")); + assert_eq!( + nested + .data + .get("response") + .and_then(|value| value.get("model")), + Some(&serde_json::json!("gpt-5")) + ); + let nested_without_flat = enforce_model( + &event(serde_json::json!({ + "type":"response.create", + "response":{"model":"also-wrong"} + })), + "gpt-5", + ); + assert!(!nested_without_flat.data.contains_key("model")); + } +} diff --git a/litellm-rust/crates/providers/src/bedrock/audio_transcription/mod.rs b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs similarity index 94% rename from litellm-rust/crates/providers/src/bedrock/audio_transcription/mod.rs rename to litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs index 7da2aa42a51..39734d844da 100644 --- a/litellm-rust/crates/providers/src/bedrock/audio_transcription/mod.rs +++ b/litellm-rust/crates/llms/src/bedrock/audio_transcription/mod.rs @@ -1,15 +1,18 @@ +use litellm_auth_aws::{ + bedrock_model_id_and_region, + constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}, + resolve_bedrock_region, +}; +use litellm_core_utils::core_helpers::json_type_name; use serde_json::{Map, Value, json}; -use crate::audio_transcription::Error; -use crate::audio_transcription::json_type_name; -use crate::audio_transcription::types::{ - AudioTranscriptionRequestData, AudioTranscriptionResponseData, +use crate::base_llm::{ + audio_transcription::transformation::{ + AudioTranscriptionAuth, AudioTranscriptionRequestData, AudioTranscriptionResponseData, + BaseAudioTranscriptionConfig, + }, + chat::transformation::Error, }; -use crate::base_llm::audio_transcription::transformation::{ - AudioTranscriptionAuth, BaseAudioTranscriptionConfig, -}; -use litellm_auth_aws::constants::{BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}; -use litellm_auth_aws::{bedrock_model_id_and_region, resolve_bedrock_region}; const SUPPORTED_PARAMS: &[&str] = &["language", "prompt", "temperature", "response_format"]; diff --git a/litellm-rust/crates/providers/src/bedrock/chat/converse_transformation.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs similarity index 93% rename from litellm-rust/crates/providers/src/bedrock/chat/converse_transformation.rs rename to litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs index 85ba3be9b07..23c6c5c61bd 100644 --- a/litellm-rust/crates/providers/src/bedrock/chat/converse_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs @@ -1,18 +1,25 @@ +use litellm_auth_aws::{ + bedrock_model_id_and_region, + constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE}, + resolve_bedrock_region, +}; +use litellm_core_utils::{ + core_helpers::{finish_reason_for, unix_now, usage_from_parts}, + prompt_templates::factory::{Conversation, TurnRole, build_conversation}, +}; +use litellm_types::{ + llms::openai::{ChatMessage, ChatMessageContent}, + utils::{ + ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, + ChatCompletionsUsage, + }, +}; use serde_json::{Map, Value, json}; use crate::base_llm::chat::transformation::{ - BaseConfig, ChatCompletionsAuth, Unsupported, unsupported_message, unsupported_param, + BaseConfig, ChatCompletionsAuth, Error, ProviderChatRequestData, ProviderChatResponseData, + Unsupported, unsupported_message, unsupported_param, }; -use crate::chat::Error; -use crate::chat::conversation::{Conversation, TurnRole, build_conversation}; -use crate::chat::response_utils::{finish_reason_for, unix_now, usage_from_parts}; -use crate::chat::types::{ - ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, - ChatCompletionsUsage, ChatMessage, ChatMessageContent, ProviderChatRequestData, - ProviderChatResponseData, -}; -use litellm_auth_aws::constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE}; -use litellm_auth_aws::{bedrock_model_id_and_region, resolve_bedrock_region}; /// Converse parameter names, post `map_openai_params`, that the Rust path can /// place verbatim in `inferenceConfig`. diff --git a/litellm-rust/crates/providers/src/bedrock/chat/mod.rs b/litellm-rust/crates/llms/src/bedrock/chat/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/bedrock/chat/mod.rs rename to litellm-rust/crates/llms/src/bedrock/chat/mod.rs diff --git a/litellm-rust/crates/providers/src/bedrock/chat/tests.rs b/litellm-rust/crates/llms/src/bedrock/chat/tests.rs similarity index 99% rename from litellm-rust/crates/providers/src/bedrock/chat/tests.rs rename to litellm-rust/crates/llms/src/bedrock/chat/tests.rs index cfa0c902096..cca5cbda41a 100644 --- a/litellm-rust/crates/providers/src/bedrock/chat/tests.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/tests.rs @@ -1,7 +1,7 @@ use serde_json::json; use super::*; -use crate::chat::Error; +use crate::base_llm::chat::transformation::Error; fn messages(value: Value) -> Vec { serde_json::from_value(value).expect("valid messages") diff --git a/litellm-rust/crates/providers/src/bedrock/mod.rs b/litellm-rust/crates/llms/src/bedrock/mod.rs similarity index 100% rename from litellm-rust/crates/providers/src/bedrock/mod.rs rename to litellm-rust/crates/llms/src/bedrock/mod.rs diff --git a/litellm-rust/crates/llms/src/cohere/mod.rs b/litellm-rust/crates/llms/src/cohere/mod.rs new file mode 100644 index 00000000000..3621ff6a2fd --- /dev/null +++ b/litellm-rust/crates/llms/src/cohere/mod.rs @@ -0,0 +1 @@ +pub mod ocr; diff --git a/litellm-rust/crates/llms/src/cohere/ocr/mod.rs b/litellm-rust/crates/llms/src/cohere/ocr/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/cohere/ocr/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/core/src/llms/cohere/ocr/transformation.rs b/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs similarity index 75% rename from litellm-rust/crates/core/src/llms/cohere/ocr/transformation.rs rename to litellm-rust/crates/llms/src/cohere/ocr/transformation.rs index 573c0b833d8..f353c22d8c4 100644 --- a/litellm-rust/crates/core/src/llms/cohere/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/cohere/ocr/transformation.rs @@ -1,42 +1,46 @@ +use litellm_core_utils::{ + call_arguments::{CallArguments, parse_options}, + serde_compat::LaxI64, + url_utils::ApiUrl, +}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use serde_with::serde_as; use crate::{ - call_arguments::{CallArguments, parse_options}, - constants::{COHERE_API_KEY_ENV, COHERE_PARSE_API_BASE}, - llms::base_llm::ocr::transformation::{BaseOcrConfig, decode_and_normalize_response}, - ocr::{ - OcrClient, + base_llm::ocr::{ document::InlineDocument, - prepare::credential_env, - types::{ - LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrPage, OcrPageImage, - OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest, + error::Error, + transformation::{ + BaseOcrConfig, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES, OcrConnection, OcrDocument, + OcrPage, OcrPageImage, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest, + credential_env, decode_and_normalize_response, decode_response_value, }, }, - serde_compat::LaxI64, - url_utils::ApiUrl, + custom_httpx::llm_http_handler::OcrClient, }; +const COHERE_PARSE_API_BASE: &str = "https://api.cohere.com"; +const COHERE_API_KEY_ENV: &str = "COHERE_API_KEY"; + const COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI: &str = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4//8/AAX+Av4N70a4AAAAAElFTkSuQmCC"; #[derive(Clone, Copy, Debug, Default, Deserialize, Serialize)] #[serde(rename_all = "lowercase")] -pub(crate) enum OutputFormat { +pub enum OutputFormat { #[default] Markdown, Blocks, } #[derive(Default, Deserialize, Serialize)] -pub(crate) struct CohereOptions { +pub struct CohereOptions { #[serde(skip_serializing_if = "Option::is_none")] pub output_format: Option, } #[derive(Deserialize, Serialize)] -pub(crate) struct CohereRequest { +pub struct CohereRequest { pub model: String, pub document: CohereParseDocument, pub output_format: String, @@ -44,13 +48,13 @@ pub(crate) struct CohereRequest { #[derive(Deserialize, Serialize)] #[serde(tag = "type")] -pub(crate) enum CohereParseDocument { +pub enum CohereParseDocument { #[serde(rename = "image_url")] ImageUrl { image_url: String }, } #[derive(Deserialize)] -pub(crate) struct CohereResponse { +pub struct CohereResponse { #[serde(default)] pages: Vec, meta: Option, @@ -85,7 +89,7 @@ struct CohereBilledUnits { } #[derive(Default)] -pub(crate) struct CohereParseConfig; +pub struct CohereParseConfig; impl BaseOcrConfig for CohereParseConfig { type OcrParams = CohereOptions; @@ -111,7 +115,7 @@ impl BaseOcrConfig for CohereParseConfig { &self, non_default_params: &CallArguments, _model: &str, - ) -> Result { + ) -> Result { Ok(parse_options(non_default_params)?) } @@ -119,7 +123,7 @@ impl BaseOcrConfig for CohereParseConfig { &self, request: &PreparedOcrRequest, _client: &OcrClient, - ) -> Result { + ) -> Result { self.resolve_headers(&request.connection, &credential_env) } @@ -128,7 +132,7 @@ impl BaseOcrConfig for CohereParseConfig { request: &PreparedOcrRequest, _optional_params: &Self::OcrParams, _environment: &Self::Environment, - ) -> Result { + ) -> Result { self.build_ocr_url( request .connection @@ -144,7 +148,7 @@ impl BaseOcrConfig for CohereParseConfig { document: OcrDocument, optional_params: &CohereOptions, _headers: &[(String, String)], - ) -> Result { + ) -> Result { let image_url = image_url(document)?; Ok(build_request(model, image_url, optional_params)) } @@ -154,12 +158,12 @@ impl BaseOcrConfig for CohereParseConfig { model: &str, raw_response: &[u8], request_format: OcrResponseFormat, - ) -> Result { + ) -> Result { decode_and_normalize_response(model, raw_response, request_format, normalize_response) } - fn validate_request_body(&self, body: &Value) -> Result<(), crate::ocr::Error> { - validate_document(&crate::ocr::prepare::body_document(body)?) + fn validate_request_body(&self, body: &Value) -> Result<(), Error> { + validate_document(&crate::custom_httpx::llm_http_handler::body_document(body)?) } } @@ -168,8 +172,9 @@ impl CohereParseConfig { &self, connection: &OcrConnection, env_lookup: &(dyn Fn(&str) -> Option + Sync), - ) -> Result, crate::ocr::Error> { - if crate::http_utils::has_header(&connection.extra_headers, "authorization") { + ) -> Result, Error> { + if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization") + { return Ok(connection.extra_headers.clone()); } let key = connection @@ -184,7 +189,7 @@ impl CohereParseConfig { .filter(|key| !key.trim().is_empty()) }) .ok_or_else(|| { - crate::ocr::Error::Auth(litellm_auth::Error::ProviderAuthentication( + Error::Auth(litellm_auth::Error::ProviderAuthentication( "Missing COHERE_API_KEY - set it in the environment or pass api_key".into(), )) })?; @@ -195,7 +200,7 @@ impl CohereParseConfig { ) } - fn build_ocr_url(&self, api_base: &str) -> Result { + fn build_ocr_url(&self, api_base: &str) -> Result { let parsed = reqwest::Url::parse(api_base).map_err(|_| invalid_api_base())?; if !matches!(parsed.scheme(), "http" | "https") { return Err(invalid_api_base()); @@ -207,35 +212,35 @@ impl CohereParseConfig { } } -pub(crate) fn validate_document(document: &OcrDocument) -> Result<(), crate::ocr::Error> { +pub fn validate_document(document: &OcrDocument) -> Result<(), Error> { let OcrDocument::ImageUrl { image_url, .. } = document else { - return Err(crate::ocr::Error::CohereImageOnly); + return Err(Error::CohereImageOnly); }; if image_url.is_empty() { - return Err(crate::ocr::Error::CohereImageOnly); + return Err(Error::CohereImageOnly); } if let Some(inline) = InlineDocument::parse(image_url)? { if !inline.mime_type().type_.eq_ignore_ascii_case("image") { - return Err(crate::ocr::Error::CohereImageOnly); + return Err(Error::CohereImageOnly); } - inline.decode(crate::constants::OCR_INLINE_MAX_BYTES)?; + inline.decode(OCR_INLINE_MAX_BYTES)?; } Ok(()) } -pub(crate) fn normalize_response( +pub fn normalize_response( model: &str, response: CohereResponse, -) -> Result { +) -> Result { let pages_processed = billed_pages(&response).map(Ok).unwrap_or_else(|| { - i64::try_from(response.pages.len()).map_err(|_| crate::ocr::Error::NumericRange("pages")) + i64::try_from(response.pages.len()).map_err(|_| Error::NumericRange("pages")) })?; let pages = response .pages .into_iter() .enumerate() .map(|(position, page)| normalize_page(page, position)) - .collect::, crate::ocr::Error>>()?; + .collect::, Error>>()?; Ok(LiteLLMOcrResponse { usage_info: Some(OcrUsageInfo { pages_processed: Some(pages_processed), @@ -245,10 +250,10 @@ pub(crate) fn normalize_response( }) } -fn image_url(document: OcrDocument) -> Result { +fn image_url(document: OcrDocument) -> Result { validate_document(&document)?; let OcrDocument::ImageUrl { image_url, .. } = document else { - return Err(crate::ocr::Error::CohereImageOnly); + return Err(Error::CohereImageOnly); }; Ok(image_url) } @@ -265,19 +270,16 @@ fn build_request(model: &str, image_url: String, params: &CohereOptions) -> Cohe } } -fn page_image( - mut image: Map, - path: &str, -) -> Result { +fn page_image(mut image: Map, path: &str) -> Result { if let Some(Value::Object(bbox)) = image.get("bounding_box") { image.insert("bbox".into(), Value::Object(bbox.clone())); } - crate::ocr::json::decode_response_value(Value::Object(image), path) + decode_response_value(Value::Object(image), path) } -fn normalize_page(page: CoherePage, position: usize) -> Result { +fn normalize_page(page: CoherePage, position: usize) -> Result { let index = page.index.map(Ok).unwrap_or_else(|| { - i64::try_from(position).map_err(|_| crate::ocr::Error::NumericRange("page index")) + i64::try_from(position).map_err(|_| Error::NumericRange("page index")) })?; let (markdown, images) = match page.markdown { Some(markdown) => { @@ -324,8 +326,8 @@ fn billed_pages(response: &CohereResponse) -> Option { response.meta.as_ref()?.billed_units.as_ref()?.pages } -fn invalid_api_base() -> crate::ocr::Error { - crate::ocr::Error::RequestField { +fn invalid_api_base() -> Error { + Error::RequestField { path: "api_base".into(), } } @@ -336,42 +338,7 @@ mod tests { use serde_json::json; use super::*; - - #[tokio::test] - async fn composed_body_preserves_native_document_fields_and_untyped_overrides() { - let request = crate::ocr::test_support::wire_request( - "cohere/parse", - "https://example.com", - json!({ - "output_format":"markdown", "timeout":30, - "extra_body":{ - "output_format": {"future":true}, - "document":{"type":"image_url","image_url":"https://example.com/a.png", - "provider_options":{"nested":[false,0,null]}} - } - }), - ); - let request = request.with_document( - serde_json::from_value(json!({ - "type":"image_url","image_url":"https://example.com/original.png" - })) - .unwrap(), - ); - let request = crate::ocr::prepare::prepare_request_for_test(request); - let http = CohereParseConfig - .prepare_request(&request, &crate::ocr::test_support::ocr_client()) - .await - .unwrap(); - let body: Value = serde_json::from_slice(http.body().unwrap().as_bytes().unwrap()).unwrap(); - assert_eq!( - body, - json!({ - "model":"parse", "output_format":{"future":true}, - "document":{"type":"image_url","image_url":"https://example.com/a.png", - "provider_options":{"nested":[false,0,null]}} - }) - ); - } + use crate::azure_ai::ocr::cohere_parse_transformation::AzureAICohereParseConfig; #[rstest] #[case::cohere(false)] @@ -382,8 +349,7 @@ mod tests { })) .unwrap(); let mapped = if azure { - crate::llms::azure_ai::ocr::cohere_parse_transformation::AzureAICohereParseConfig - .map_ocr_params(&arguments, "parse") + AzureAICohereParseConfig.map_ocr_params(&arguments, "parse") } else { CohereParseConfig.map_ocr_params(&arguments, "parse") } @@ -401,7 +367,7 @@ mod tests { let invalid = serde_json::from_value(json!({"output_format":"html"})).unwrap(); assert!(matches!( CohereParseConfig.map_ocr_params(&invalid, "parse"), - Err(crate::ocr::Error::RequestField { path }) + Err(Error::RequestField { path }) if path == "optional_params.output_format" )); } @@ -463,7 +429,7 @@ mod tests { .unwrap(); assert!(matches!( normalize_response("parse", response).unwrap_err(), - crate::ocr::Error::ResponseField { path } + Error::ResponseField { path } if path == "pages[0].markdown.images[0].image_base64" )); } @@ -499,33 +465,6 @@ mod tests { ); } - #[tokio::test] - async fn explicit_null_options_use_defaults_before_http() { - let request = crate::ocr::test_support::wire_request( - "cohere/parse", - "https://example.com", - json!({"output_format":null,"req_format":null}), - ); - let request = request.with_document( - serde_json::from_value( - json!({"type":"image_url","image_url":"https://example.com/a.png"}), - ) - .unwrap(), - ); - assert_eq!( - request.response_format().unwrap(), - crate::ocr::types::OcrResponseFormat::Litellm - ); - let request = crate::ocr::prepare::prepare_request_for_test(request); - let http = CohereParseConfig - .prepare_request(&request, &crate::ocr::test_support::ocr_client()) - .await - .unwrap(); - let body: Value = serde_json::from_slice(http.body().unwrap().as_bytes().unwrap()).unwrap(); - assert_eq!(body["output_format"], "markdown"); - assert!(body.get("req_format").is_none()); - } - #[rstest] fn response_normalizes_markdown_images_blocks_and_billed_pages() { let payload = json!({ @@ -619,10 +558,10 @@ mod tests { #[rstest] fn response_types_documented_block_variants( #[values( - crate::ocr::types::OcrResponseFormat::Litellm, - crate::ocr::types::OcrResponseFormat::Native + crate::base_llm::ocr::transformation::OcrResponseFormat::Litellm, + crate::base_llm::ocr::transformation::OcrResponseFormat::Native )] - response_format: crate::ocr::types::OcrResponseFormat, + response_format: crate::base_llm::ocr::transformation::OcrResponseFormat, ) { let payload = json!({ "pages": [{ @@ -692,10 +631,10 @@ mod tests { Some(1) ); match response_format { - crate::ocr::types::OcrResponseFormat::Litellm => { + crate::base_llm::ocr::transformation::OcrResponseFormat::Litellm => { assert!(normalized.provider_native_response.is_none()); } - crate::ocr::types::OcrResponseFormat::Native => { + crate::base_llm::ocr::transformation::OcrResponseFormat::Native => { assert_eq!( normalized.provider_native_response.as_ref(), payload.as_object() @@ -715,7 +654,7 @@ mod tests { fn request_requires_image(#[case] value: Value) { assert!(matches!( validate_document(&serde_json::from_value(value).unwrap()), - Err(crate::ocr::Error::CohereImageOnly) + Err(Error::CohereImageOnly) )); } @@ -784,7 +723,7 @@ mod tests { }, &|_| None, ), - Err(crate::ocr::Error::Auth(_)) + Err(Error::Auth(_)) )); } @@ -810,61 +749,4 @@ mod tests { assert!(error.to_string().contains(COHERE_API_KEY_ENV), "{error}"); } - - #[rstest] - #[case::cohere("cohere/parse-v5.0", "POST /v2/parse ")] - #[case::azure_ai("azure_ai/Cohere-parse-v5.0", "POST /providers/cohere/v2/parse ")] - #[tokio::test] - async fn route_sends_image_to_its_parse_endpoint_with_the_bearer_key( - #[case] model: &str, - #[case] request_line: &str, - ) { - use crate::ocr::test_support::{MockResponse, header, mock_server, perform_ocr}; - - let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; - let request = crate::ocr::test_support::wire_request(model, &base, json!({})) - .with_document( - serde_json::from_value::( - json!({"type":"image_url","image_url":"data:image/png;base64,YWJj"}), - ) - .unwrap() - .into(), - ); - - perform_ocr(request).await.unwrap(); - server.await.unwrap(); - - let requests = seen.lock().unwrap(); - assert_eq!(requests.len(), 1); - assert!(requests[0].starts_with(request_line), "{}", requests[0]); - assert_eq!( - header(&requests[0], "authorization"), - Some("Bearer test-key") - ); - } - - #[rstest] - #[tokio::test] - async fn route_rejects_non_image_document_without_a_request( - #[values("cohere/parse-v5.0", "azure_ai/Cohere-parse-v5.0")] model: &str, - ) { - use crate::ocr::test_support::{MockResponse, mock_server, perform_ocr}; - - let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await; - - let error = perform_ocr(crate::ocr::test_support::wire_request( - model, - &base, - json!({}), - )) - .await - .unwrap_err(); - server.abort(); - - assert!( - matches!(error, crate::ocr::Error::CohereImageOnly), - "{error:?}" - ); - assert!(seen.lock().unwrap().is_empty()); - } } diff --git a/litellm-rust/crates/core/src/http_utils.rs b/litellm-rust/crates/llms/src/custom_httpx/http_handler.rs similarity index 92% rename from litellm-rust/crates/core/src/http_utils.rs rename to litellm-rust/crates/llms/src/custom_httpx/http_handler.rs index 060559322ea..e629be37336 100644 --- a/litellm-rust/crates/core/src/http_utils.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/http_handler.rs @@ -6,15 +6,18 @@ pub struct HeaderError { pub actual: &'static str, } +use litellm_core_utils::core_helpers::json_type_name; use serde_json::{Map, Value}; -use crate::constants::UPSTREAM_ERROR_BODY_MAX_CHARS; +/// Max characters of an upstream error body echoed across the call boundary +/// before truncation, so provider bodies are bounded and data-minimized. +const UPSTREAM_ERROR_BODY_MAX_CHARS: usize = 256; #[allow( dead_code, reason = "used by the OCR architecture in the next stacked PR" )] -pub(crate) enum HeaderPolicy<'a> { +pub enum HeaderPolicy<'a> { All, Only(&'a [&'a str]), Except(&'a [&'a str]), @@ -24,7 +27,7 @@ pub(crate) enum HeaderPolicy<'a> { dead_code, reason = "used by the OCR architecture in the next stacked PR" )] -pub(crate) fn with_headers( +pub fn with_headers( builder: reqwest::RequestBuilder, headers: &[(String, String)], policy: HeaderPolicy<'_>, @@ -108,9 +111,7 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool { dead_code, reason = "used by the OCR architecture in the next stacked PR" )] -pub(crate) fn deserialize_optional_param<'de, D, T>( - deserializer: D, -) -> Result>, D::Error> +pub fn deserialize_optional_param<'de, D, T>(deserializer: D) -> Result>, D::Error> where D: serde::Deserializer<'de>, T: serde::Deserialize<'de>, @@ -118,17 +119,6 @@ where as serde::Deserialize>::deserialize(deserializer).map(Some) } -pub fn json_type_name(value: &serde_json::Value) -> &'static str { - match value { - serde_json::Value::Null => "null", - serde_json::Value::Bool(_) => "bool", - serde_json::Value::Number(_) => "number", - serde_json::Value::String(_) => "string", - serde_json::Value::Array(_) => "array", - serde_json::Value::Object(_) => "object", - } -} - #[cfg(test)] mod tests { use serde_json::json; diff --git a/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs b/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs new file mode 100644 index 00000000000..e93ddee3c50 --- /dev/null +++ b/litellm-rust/crates/llms/src/custom_httpx/llm_http_handler.rs @@ -0,0 +1,343 @@ +use std::{sync::OnceLock, time::Duration}; + +use bytes::{Bytes, BytesMut}; +use futures_util::future::BoxFuture; +use litellm_auth_gcp::VertexAuth; +use litellm_callbacks::event::{Passthrough, WireRequest}; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::{Map, Value}; + +use crate::{ + base_llm::ocr::{ + error::Error, + transformation::{ + BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OCR_CONNECT_TIMEOUT_SECS, + OcrDocument, OcrResponseContext, PreparedOcrRequest, decode_request_value, + decode_response, + }, + }, + custom_httpx::{ + http_handler::{HeaderPolicy, execute_http_request, with_headers}, + media::MediaFetcher, + transport, + }, +}; + +/// The route's view of one call, handed to provider code that has to reach the +/// caller's hooks mid-flight (guardrails on the outgoing body, raw response events). +pub trait CallHooks: Send + Sync { + fn before_send( + &self, + wire: WireRequest, + passthrough_fields: Passthrough, + ) -> BoxFuture<'_, Result>; + + fn response_received<'a>(&'a self, body: &'a [u8]) -> BoxFuture<'a, Result<(), E>>; +} + +#[derive(Clone)] +pub struct OcrClient { + provider_http: reqwest::Client, + polling_http: reqwest::Client, + document_fetcher: MediaFetcher, + vertex_auth: VertexAuth, +} + +impl OcrClient { + pub fn new(provider_http: reqwest::Client) -> Result { + let document_fetcher = MediaFetcher::new().map_err(transport::Error::from)?; + Ok(Self { + provider_http, + polling_http: no_redirect_http()?, + document_fetcher, + vertex_auth: VertexAuth::default(), + }) + } + + pub fn shared() -> Result { + static CLIENT: OnceLock> = OnceLock::new(); + let client = CLIENT + .get_or_init(|| { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(OCR_CONNECT_TIMEOUT_SECS)) + .build() + .map_err(transport::Error::from) + .and_then(OcrClient::new) + }) + .clone()?; + Ok(client) + } + + pub fn provider_http(&self) -> &reqwest::Client { + &self.provider_http + } + + pub fn polling_http(&self) -> &reqwest::Client { + &self.polling_http + } + + pub fn document_fetcher(&self) -> &MediaFetcher { + &self.document_fetcher + } + + pub fn vertex_auth(&self) -> &VertexAuth { + &self.vertex_auth + } + + #[cfg(any(test, feature = "test-support"))] + pub fn for_test(provider_http: reqwest::Client, document_http: reqwest::Client) -> Self { + Self { + provider_http, + polling_http: no_redirect_http().expect("test polling client builds"), + document_fetcher: MediaFetcher::for_test(document_http), + vertex_auth: VertexAuth::default(), + } + } +} + +fn no_redirect_http() -> Result { + reqwest::Client::builder() + .connect_timeout(Duration::from_secs(OCR_CONNECT_TIMEOUT_SECS)) + .redirect(reqwest::redirect::Policy::none()) + .build() + .map_err(transport::Error::from) +} + +/// Rust counterpart of `BaseLLMHTTPHandler.async_ocr`: prepare the provider request, +/// send it, and hand the response to the config for normalization. +pub async fn ocr( + config: &C, + client: &OcrClient, + request: &PreparedOcrRequest, + hooks: &dyn CallHooks, +) -> Result { + let http = config.prepare_request(request, client, hooks).await?; + let url = http.url().to_string(); + let headers = request_headers(&http)?; + let response = execute_http_request(client.provider_http(), http) + .await + .map_err(transport_error)?; + if !response.status().is_success() { + let headers = response + .headers() + .iter() + .filter_map(|(name, value)| { + value + .to_str() + .ok() + .map(|value| (name.to_string(), value.to_string())) + }) + .collect(); + return match read_response_bytes(response, request.connection.max_response_bytes).await { + Err(Error::Transport(transport::Error::Http { status, body })) => { + Err(config.get_error_class(body, status, headers)) + } + Err(error) => Err(error), + Ok(_) => unreachable!("non-success response produces an HTTP error"), + }; + } + let context = OcrResponseContext { + client, + connection: &request.connection, + hooks, + request_format: request.response_format()?, + url: &url, + headers: &headers, + }; + config + .async_transform_ocr_response(&request.model, response, context) + .await +} + +fn request_headers(request: &reqwest::Request) -> Result, Error> { + request + .headers() + .iter() + .map(|(name, value)| { + value + .to_str() + .map(|value| (name.to_string(), value.to_string())) + .map_err(|_| Error::RequestField { + path: "headers".into(), + }) + }) + .collect() +} + +pub async fn read_json_response( + response: reqwest::Response, + native: bool, + max_response_bytes: usize, +) -> Result, Error> { + let bytes = read_response_bytes(response, max_response_bytes).await?; + decode_response(&bytes, native) +} + +pub async fn read_response_bytes( + mut response: reqwest::Response, + limit: usize, +) -> Result { + let status = response.status(); + if status.is_success() + && response + .content_length() + .is_some_and(|length| length > limit as u64) + { + return Err(Error::TooLarge { limit }); + } + let mut bytes = BytesMut::new(); + while let Some(chunk) = response.chunk().await.map_err(transport_error)? { + let remaining = limit.saturating_sub(bytes.len()); + if status.is_success() && chunk.len() > remaining { + return Err(Error::TooLarge { limit }); + } + bytes.extend_from_slice(&chunk[..chunk.len().min(remaining)]); + if !status.is_success() && bytes.len() == limit { + break; + } + } + if !status.is_success() { + return Err(transport::Error::Http { + status: status.as_u16(), + body: String::from_utf8_lossy(&bytes).into_owned(), + } + .into()); + } + Ok(bytes.freeze()) +} + +pub fn transport_error(error: reqwest::Error) -> Error { + if error.is_timeout() { + return Error::Transport(transport::Error::Http { + status: 408, + body: "OCR request timed out".into(), + }); + } + transport::Error::from(error).into() +} + +pub async fn transform_request_body( + config: &C, + client: &OcrClient, + request: &PreparedOcrRequest, + url: &str, + headers: &[(String, String)], + body: B, + hooks: &dyn CallHooks, +) -> Result { + let composed = litellm_core_utils::call_arguments::compose_body( + &request.optional_params, + &body, + config.get_supported_ocr_params(&request.model), + )?; + config.validate_request_body(&composed)?; + let passthrough_fields = Passthrough::unchanged(&caller_inputs(request)?, &composed); + let changed = hooks + .before_send(wire_request(url, headers, composed), passthrough_fields) + .await?; + if !changed.body.is_object() { + return Err(Error::RequestField { + path: "guardrail.body".into(), + }); + } + config.validate_request_body(&changed.body)?; + build_http_request(client, request, url, &changed.headers, &changed.body) +} + +fn wire_request(url: &str, headers: &[(String, String)], body: Value) -> WireRequest { + WireRequest { + url: url.into(), + headers: headers.to_vec(), + body, + } +} + +fn caller_inputs(request: &PreparedOcrRequest) -> Result, Error> { + let document = request + .caller_document + .then(|| serde_json::to_value(&request.document)) + .transpose() + .map_err(|_| Error::RequestField { + path: "document".into(), + })?; + let params: Map = request.optional_params.clone().into(); + Ok(params + .into_iter() + .chain(document.map(|document| ("document".to_string(), document))) + .collect()) +} + +pub fn build_http_request( + client: &OcrClient, + request: &PreparedOcrRequest, + url: &str, + headers: &[(String, String)], + body: &B, +) -> Result { + let builder = client + .provider_http() + .post(url) + .json(body) + .timeout(request.connection.timeout); + with_headers(builder, headers, HeaderPolicy::All) + .build() + .map_err(transport::Error::from) + .map_err(Error::from) +} + +pub async fn guardrail_document( + request: &PreparedOcrRequest, + url: &str, + headers: &[(String, String)], + hooks: &dyn CallHooks, +) -> Result<(OcrDocument, Vec<(String, String)>), Error> { + let body = serde_json::to_value(&request.document).map_err(|_| Error::RequestField { + path: "document".into(), + })?; + let changed = hooks + .before_send(wire_request(url, headers, body), Passthrough::default()) + .await?; + let document = decode_request_value(changed.body, "guardrail.document")?; + Ok((document, changed.headers)) +} + +pub fn body_document(body: &Value) -> Result { + let document = body + .get("document") + .and_then(Value::as_object) + .ok_or_else(|| Error::RequestField { + path: "body.document".into(), + })?; + let source = document + .iter() + .filter(|(name, _)| matches!(name.as_str(), "type" | "image_url" | "document_url")) + .map(|(name, value)| (name.clone(), value.clone())) + .collect(); + decode_request_value(Value::Object(source), "body.document") +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn request_timeout_has_an_http_408_status() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { + let _connection = listener.accept().await.unwrap(); + tokio::time::sleep(Duration::from_secs(1)).await; + }); + let error = reqwest::Client::new() + .get(format!("http://{address}")) + .timeout(Duration::from_millis(10)) + .send() + .await + .unwrap_err(); + assert!(matches!( + transport_error(error), + Error::Transport(transport::Error::Http { status: 408, .. }) + )); + server.abort(); + } +} diff --git a/litellm-rust/crates/core/src/media.rs b/litellm-rust/crates/llms/src/custom_httpx/media.rs similarity index 95% rename from litellm-rust/crates/core/src/media.rs rename to litellm-rust/crates/llms/src/custom_httpx/media.rs index 3a6579bb0a6..0b7fa30e34b 100644 --- a/litellm-rust/crates/core/src/media.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/media.rs @@ -12,10 +12,10 @@ use reqwest::{ dns::{Addrs, Name, Resolve, Resolving}, }; -use crate::constants::MEDIA_CONNECT_TIMEOUT_SECS; +const MEDIA_CONNECT_TIMEOUT_SECS: u64 = 10; #[derive(Debug, thiserror::Error)] -pub(crate) enum Error { +pub enum Error { #[error("media URL rejected by network policy")] BlockedUrl, #[error("media download is disabled")] @@ -33,11 +33,11 @@ pub(crate) enum Error { #[error("media download timed out")] Timeout, #[error("{0}")] - Transport(#[from] crate::transport::Error), + Transport(#[from] crate::custom_httpx::transport::Error), } #[derive(Clone)] -pub(crate) struct MediaFetcher { +pub struct MediaFetcher { client: reqwest::Client, address_resolver: Arc, allow_private_network: bool, @@ -50,20 +50,20 @@ trait AddressResolver: Send + Sync { } #[derive(Clone, Copy)] -pub(crate) struct DownloadPolicy { - pub(crate) timeout: Duration, - pub(crate) max_bytes: u64, - pub(crate) max_redirects: usize, +pub struct DownloadPolicy { + pub timeout: Duration, + pub max_bytes: u64, + pub max_redirects: usize, } #[derive(Debug)] -pub(crate) struct DownloadedMedia { - pub(crate) bytes: Vec, - pub(crate) content_type: String, +pub struct DownloadedMedia { + pub bytes: Vec, + pub content_type: String, } impl MediaFetcher { - pub(crate) fn new() -> Result { + pub fn new() -> Result { Self::with_resolvers(Arc::new(PublicDnsResolver), Arc::new(SystemAddressResolver)) } @@ -87,8 +87,8 @@ impl MediaFetcher { }) } - #[cfg(test)] - pub(crate) fn for_test(client: reqwest::Client) -> Self { + #[cfg(any(test, feature = "test-support"))] + pub fn for_test(client: reqwest::Client) -> Self { Self { client, address_resolver: Arc::new(AllowPrivateResolver), @@ -96,11 +96,7 @@ impl MediaFetcher { } } - pub(crate) async fn fetch( - &self, - url: Url, - policy: DownloadPolicy, - ) -> Result { + pub async fn fetch(&self, url: Url, policy: DownloadPolicy) -> Result { if policy.max_bytes == 0 { return Err(Error::DownloadDisabled); } @@ -122,7 +118,7 @@ impl MediaFetcher { .get(url.clone()) .send() .await - .map_err(crate::transport::Error::from)?; + .map_err(crate::custom_httpx::transport::Error::from)?; if response.status().is_redirection() { if redirects_followed == policy.max_redirects { return Err(Error::TooManyRedirects); @@ -153,7 +149,7 @@ impl MediaFetcher { while let Some(chunk) = response .chunk() .await - .map_err(crate::transport::Error::from)? + .map_err(crate::custom_httpx::transport::Error::from)? { enforce_download_size(bytes.len() as u64 + chunk.len() as u64, policy.max_bytes)?; bytes.extend_from_slice(&chunk); @@ -184,7 +180,7 @@ impl MediaFetcher { .address_resolver .resolve(host, port) .await - .map_err(|error| crate::transport::Error::Network(error.to_string()))?; + .map_err(|error| crate::custom_httpx::transport::Error::Network(error.to_string()))?; validate_addresses(&addresses) } } @@ -254,10 +250,10 @@ impl AddressResolver for SystemAddressResolver { } } -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] struct AllowPrivateResolver; -#[cfg(test)] +#[cfg(any(test, feature = "test-support"))] impl AddressResolver for AllowPrivateResolver { fn resolve<'a>(&'a self, _host: &'a str, port: u16) -> AddressResolution<'a> { Box::pin(async move { Ok(vec![SocketAddr::from(([8, 8, 8, 8], port))]) }) diff --git a/litellm-rust/crates/llms/src/custom_httpx/mod.rs b/litellm-rust/crates/llms/src/custom_httpx/mod.rs new file mode 100644 index 00000000000..057cb796c09 --- /dev/null +++ b/litellm-rust/crates/llms/src/custom_httpx/mod.rs @@ -0,0 +1,4 @@ +pub mod http_handler; +pub mod llm_http_handler; +pub mod media; +pub mod transport; diff --git a/litellm-rust/crates/core/src/transport/error.rs b/litellm-rust/crates/llms/src/custom_httpx/transport.rs similarity index 86% rename from litellm-rust/crates/core/src/transport/error.rs rename to litellm-rust/crates/llms/src/custom_httpx/transport.rs index eff15365ea8..172dd96476a 100644 --- a/litellm-rust/crates/core/src/transport/error.rs +++ b/litellm-rust/crates/llms/src/custom_httpx/transport.rs @@ -38,8 +38,11 @@ mod tests { .send() .await .expect_err("invalid port"); - let error = crate::transport::Error::from_reqwest_before_dispatch(error); - assert!(matches!(error, crate::transport::Error::Connect(_))); + let error = crate::custom_httpx::transport::Error::from_reqwest_before_dispatch(error); + assert!(matches!( + error, + crate::custom_httpx::transport::Error::Connect(_) + )); assert!(!error.to_string().contains("secret")); assert!(!error.to_string().contains("private")); } @@ -68,8 +71,8 @@ mod tests { let error = response.expect_err("server does not respond"); assert!(error.is_timeout()); assert!(matches!( - crate::transport::Error::from_reqwest_before_dispatch(error), - crate::transport::Error::Network(_) + crate::custom_httpx::transport::Error::from_reqwest_before_dispatch(error), + crate::custom_httpx::transport::Error::Network(_) )); } } diff --git a/litellm-rust/crates/llms/src/lib.rs b/litellm-rust/crates/llms/src/lib.rs new file mode 100644 index 00000000000..884fa739992 --- /dev/null +++ b/litellm-rust/crates/llms/src/lib.rs @@ -0,0 +1,10 @@ +pub mod anthropic; +pub mod azure_ai; +pub mod base_llm; +pub mod bedrock; +pub mod cohere; +pub mod custom_httpx; +pub mod mistral; +pub mod openai; +pub mod reducto; +pub mod vertex_ai; diff --git a/litellm-rust/crates/llms/src/mistral/mod.rs b/litellm-rust/crates/llms/src/mistral/mod.rs new file mode 100644 index 00000000000..3621ff6a2fd --- /dev/null +++ b/litellm-rust/crates/llms/src/mistral/mod.rs @@ -0,0 +1 @@ +pub mod ocr; diff --git a/litellm-rust/crates/llms/src/mistral/ocr/mod.rs b/litellm-rust/crates/llms/src/mistral/ocr/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/mistral/ocr/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/core/src/llms/mistral/ocr/transformation.rs b/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs similarity index 91% rename from litellm-rust/crates/core/src/llms/mistral/ocr/transformation.rs rename to litellm-rust/crates/llms/src/mistral/ocr/transformation.rs index dac1ed7c68f..c2038d0552d 100644 --- a/litellm-rust/crates/core/src/llms/mistral/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/mistral/ocr/transformation.rs @@ -1,26 +1,25 @@ +use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl}; use serde::{Deserialize, Serialize}; use serde_json::Value; use crate::{ - call_arguments::CallArguments, - constants::MISTRAL_OCR_API_BASE, - llms::base_llm::ocr::transformation::{BaseOcrConfig, decode_and_normalize_response}, - ocr::{ - OcrClient, - prepare::credential_env, - types::{ - LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrPage, OcrResponseFormat, - OcrUsageInfo, PreparedOcrRequest, + base_llm::ocr::{ + error::Error, + transformation::{ + BaseOcrConfig, LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrPage, + OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest, credential_env, + decode_and_normalize_response, }, }, - params::OpaqueParams, - url_utils::ApiUrl, + custom_httpx::llm_http_handler::OcrClient, }; +const MISTRAL_OCR_API_BASE: &str = "https://api.mistral.ai/v1"; + const MISTRAL_OCR_API_KEY_ENV_VAR: &str = "MISTRAL_API_KEY"; #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct MistralOcrRequest { +pub struct MistralOcrRequest { pub model: String, pub document: OcrDocument, #[serde(flatten)] @@ -28,7 +27,7 @@ pub(crate) struct MistralOcrRequest { } #[derive(Clone, Debug, Default, Deserialize)] -pub(crate) struct MistralOcrResponse { +pub struct MistralOcrResponse { #[serde(default)] pub pages: Vec, #[serde( @@ -44,7 +43,7 @@ pub(crate) struct MistralOcrResponse { } #[derive(Clone, Debug, Default)] -pub(crate) struct MistralOcrConfig; +pub struct MistralOcrConfig; impl BaseOcrConfig for MistralOcrConfig { type OcrParams = OpaqueParams; @@ -77,7 +76,7 @@ impl BaseOcrConfig for MistralOcrConfig { &self, non_default_params: &CallArguments, model: &str, - ) -> Result { + ) -> Result { Ok(non_default_params .select(self.get_supported_ocr_params(model)) .into()) @@ -87,7 +86,7 @@ impl BaseOcrConfig for MistralOcrConfig { &self, request: &PreparedOcrRequest, _client: &OcrClient, - ) -> Result { + ) -> Result { self.resolve_headers(&request.connection, &credential_env) } @@ -96,7 +95,7 @@ impl BaseOcrConfig for MistralOcrConfig { request: &PreparedOcrRequest, _optional_params: &Self::OcrParams, _environment: &Self::Environment, - ) -> Result { + ) -> Result { self.build_ocr_url(request.connection.api_base.as_deref()) } @@ -106,7 +105,7 @@ impl BaseOcrConfig for MistralOcrConfig { document: OcrDocument, optional_params: &OpaqueParams, _headers: &[(String, String)], - ) -> Result { + ) -> Result { Ok(MistralOcrRequest { model: model.to_string(), document, @@ -119,7 +118,7 @@ impl BaseOcrConfig for MistralOcrConfig { model: &str, raw_response: &[u8], request_format: OcrResponseFormat, - ) -> Result { + ) -> Result { decode_and_normalize_response(model, raw_response, request_format, normalize_response) } } @@ -129,8 +128,9 @@ impl MistralOcrConfig { &self, connection: &OcrConnection, env_lookup: &(dyn Fn(&str) -> Option + Sync), - ) -> Result, crate::ocr::Error> { - if crate::http_utils::has_header(&connection.extra_headers, "authorization") { + ) -> Result, Error> { + if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization") + { return Ok(connection.extra_headers.clone()); } let api_key = connection @@ -155,7 +155,7 @@ impl MistralOcrConfig { ) } - fn build_ocr_url(&self, api_base: Option<&str>) -> Result { + fn build_ocr_url(&self, api_base: Option<&str>) -> Result { let base = api_base .map(str::trim) .filter(|base| !base.is_empty()) @@ -163,20 +163,20 @@ impl MistralOcrConfig { ApiUrl::parse(base) .and_then(|url| url.complete_path(&["v1", "ocr"])) .map(|url| url.into_string()) - .map_err(|_| crate::ocr::Error::RequestField { + .map_err(|_| Error::RequestField { path: "api_base".into(), }) } } -pub(crate) fn normalize_response( +pub fn normalize_response( model: &str, response: MistralOcrResponse, -) -> Result { +) -> Result { let model = match response.model { Some(Some(model)) => model, Some(None) => { - return Err(crate::ocr::Error::ResponseField { + return Err(Error::ResponseField { path: "model".into(), }); } @@ -196,6 +196,7 @@ mod tests { use serde_json::{Value, json}; use super::*; + use crate::base_llm::ocr::transformation::decode_response; #[fixture] fn document() -> OcrDocument { @@ -222,7 +223,7 @@ mod tests { let response = serde_json::from_value(json!({"model":null})).unwrap(); assert!(matches!( normalize_response("fallback", response).unwrap_err(), - crate::ocr::Error::ResponseField { path } if path == "model" + Error::ResponseField { path } if path == "model" )); } @@ -249,14 +250,12 @@ mod tests { #[case] payload: Value, #[case] path: &str, ) { - let error = crate::ocr::json::decode_response::( - &serde_json::to_vec(&payload).unwrap(), - false, - ) - .unwrap_err(); + let error = + decode_response::(&serde_json::to_vec(&payload).unwrap(), false) + .unwrap_err(); assert!(matches!( error, - crate::ocr::Error::ResponseField { path: actual } if actual == path + Error::ResponseField { path: actual } if actual == path )); } @@ -318,7 +317,11 @@ mod tests { fn raw_response_transform_keeps_native_payload_separate_from_typed_normalization() { let raw = br#"{"pages":[{"index":"2","markdown":"text"}],"provider_extension":false}"#; let response = MistralOcrConfig - .transform_ocr_response("model", raw, crate::ocr::types::OcrResponseFormat::Native) + .transform_ocr_response( + "model", + raw, + crate::base_llm::ocr::transformation::OcrResponseFormat::Native, + ) .unwrap(); assert_eq!(response.pages[0].index, 2); let native = response.provider_native_response.unwrap(); @@ -642,12 +645,10 @@ mod tests { fn environment_rejects_missing_key(connection: OcrConnection) { assert!(matches!( MistralOcrConfig.resolve_headers(&connection, &|_| None), - Err(crate::ocr::Error::Auth( - litellm_auth::Error::MissingApiKey { - provider: "Mistral", - environment_variable: MISTRAL_OCR_API_KEY_ENV_VAR, - } - )) + Err(Error::Auth(litellm_auth::Error::MissingApiKey { + provider: "Mistral", + environment_variable: MISTRAL_OCR_API_KEY_ENV_VAR, + })) )); } } diff --git a/litellm-rust/crates/core/src/llms/openai/mod.rs b/litellm-rust/crates/llms/src/openai/mod.rs similarity index 100% rename from litellm-rust/crates/core/src/llms/openai/mod.rs rename to litellm-rust/crates/llms/src/openai/mod.rs diff --git a/litellm-rust/crates/llms/src/openai/responses/mod.rs b/litellm-rust/crates/llms/src/openai/responses/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/openai/responses/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/core/src/llms/openai/responses/transformation.rs b/litellm-rust/crates/llms/src/openai/responses/transformation.rs similarity index 84% rename from litellm-rust/crates/core/src/llms/openai/responses/transformation.rs rename to litellm-rust/crates/llms/src/openai/responses/transformation.rs index 2c8916b6806..f01ec4ad146 100644 --- a/litellm-rust/crates/core/src/llms/openai/responses/transformation.rs +++ b/litellm-rust/crates/llms/src/openai/responses/transformation.rs @@ -1,7 +1,8 @@ -use crate::responses::{ - Error, - types::{ResponsesWsEvent, ResponsesWsTransformResult}, - websocket::{ResponsesWebSocketProviderConfig, enforce_model}, +use litellm_types::responses::streaming_websocket::{ResponsesWsEvent, ResponsesWsTransformResult}; + +use crate::base_llm::{ + chat::transformation::Error, + responses::transformation::{ResponsesWebSocketProviderConfig, enforce_model}, }; pub struct OpenAiResponsesApiConfig; diff --git a/litellm-rust/crates/llms/src/reducto/mod.rs b/litellm-rust/crates/llms/src/reducto/mod.rs new file mode 100644 index 00000000000..3621ff6a2fd --- /dev/null +++ b/litellm-rust/crates/llms/src/reducto/mod.rs @@ -0,0 +1 @@ +pub mod ocr; diff --git a/litellm-rust/crates/llms/src/reducto/ocr/mod.rs b/litellm-rust/crates/llms/src/reducto/ocr/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/reducto/ocr/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/core/src/llms/reducto/ocr/transformation.rs b/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs similarity index 57% rename from litellm-rust/crates/core/src/llms/reducto/ocr/transformation.rs rename to litellm-rust/crates/llms/src/reducto/ocr/transformation.rs index 4c5323ef50e..ca2bae9c3bb 100644 --- a/litellm-rust/crates/core/src/llms/reducto/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/reducto/ocr/transformation.rs @@ -1,50 +1,55 @@ use std::collections::BTreeMap; +use litellm_core_utils::{ + call_arguments::{CallArguments, compose_body}, + params::OpaqueParams, + url_utils::ApiUrl, +}; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::{Map, Value, json}; use crate::{ - call_arguments::{CallArguments, compose_body}, - constants::{REDUCTO_API_BASE, REDUCTO_API_KEY_ENV, REDUCTO_ID_PREFIX}, - llms::base_llm::ocr::transformation::{ - BaseOcrConfig, OcrRequestContext, decode_and_normalize_response, - }, - ocr::{ - OcrClient, + base_llm::ocr::{ document::InlineDocument, - prepare::{build_http_request, credential_env, guardrail_document}, - types::{ - LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrPage, OcrResponseFormat, - OcrUsageInfo, PreparedOcrRequest, + error::Error, + transformation::{ + BaseOcrConfig, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES, OcrConnection, OcrDocument, + OcrPage, OcrRequestContext, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest, + credential_env, decode_and_normalize_response, }, }, - params::OpaqueParams, - url_utils::ApiUrl, + custom_httpx::llm_http_handler::{ + CallHooks, OcrClient, build_http_request, guardrail_document, + }, }; +const REDUCTO_API_BASE: &str = "https://platform.reducto.ai"; +const REDUCTO_API_KEY_ENV: &str = "REDUCTO_API_KEY"; +const REDUCTO_ID_PREFIX: &str = "reducto://"; + #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(transparent)] -pub(crate) struct ReductoFileId(String); +pub struct ReductoFileId(String); -pub(crate) type ReductoV3Params = OpaqueParams; -pub(crate) type ReductoLegacyParams = OpaqueParams; +pub type ReductoV3Params = OpaqueParams; +pub type ReductoLegacyParams = OpaqueParams; #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct ReductoV3Request { +pub struct ReductoV3Request { pub input: ReductoFileId, #[serde(flatten)] pub params: ReductoV3Params, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct ReductoLegacyRequest { +pub struct ReductoLegacyRequest { pub document_url: ReductoFileId, #[serde(skip_serializing_if = "Option::is_none")] pub options: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct ReductoLegacyOptions { +pub struct ReductoLegacyOptions { pub enhance: Value, } @@ -54,7 +59,7 @@ struct ReductoUploadResponse { } #[derive(Clone, Debug, Deserialize)] -pub(crate) struct ReductoResponse { +pub struct ReductoResponse { #[serde(default, deserialize_with = "present_nullable")] result: Option>, usage: Option, @@ -70,9 +75,9 @@ struct ReductoResult { #[serde_with::serde_as] #[derive(Clone, Debug, Default, Deserialize)] struct ReductoUsage { - #[serde_as(deserialize_as = "Option")] + #[serde_as(deserialize_as = "Option")] pub num_pages: Option, - #[serde_as(deserialize_as = "Option")] + #[serde_as(deserialize_as = "Option")] pub credits: Option, } @@ -83,7 +88,7 @@ struct ReductoChunk { } #[derive(Clone, Debug)] -pub(crate) struct ReductoParseV3Config; +pub struct ReductoParseV3Config; impl BaseOcrConfig for ReductoParseV3Config { type OcrParams = ReductoV3Params; @@ -98,7 +103,7 @@ impl BaseOcrConfig for ReductoParseV3Config { &self, non_default_params: &CallArguments, model: &str, - ) -> Result { + ) -> Result { Ok(non_default_params .select(self.get_supported_ocr_params(model)) .into()) @@ -108,7 +113,7 @@ impl BaseOcrConfig for ReductoParseV3Config { &self, request: &PreparedOcrRequest, _client: &OcrClient, - ) -> Result { + ) -> Result { resolve_headers(&request.connection, &credential_env) } @@ -117,7 +122,7 @@ impl BaseOcrConfig for ReductoParseV3Config { request: &PreparedOcrRequest, _optional_params: &Self::OcrParams, _environment: &Self::Environment, - ) -> Result { + ) -> Result { build_ocr_url(request.connection.api_base.as_deref()) } @@ -127,7 +132,7 @@ impl BaseOcrConfig for ReductoParseV3Config { document: OcrDocument, optional_params: &Self::OcrParams, _headers: &[(String, String)], - ) -> Result { + ) -> Result { Ok(ReductoV3Request { input: uploaded_file_id(document)?, params: optional_params.clone(), @@ -141,7 +146,7 @@ impl BaseOcrConfig for ReductoParseV3Config { optional_params: &ReductoV3Params, headers: &[(String, String)], context: OcrRequestContext<'_>, - ) -> Result { + ) -> Result { let file_id = ensure_file_id_async(document, headers, context).await?; Ok(ReductoV3Request { input: file_id, @@ -154,7 +159,7 @@ impl BaseOcrConfig for ReductoParseV3Config { model: &str, raw_response: &[u8], request_format: OcrResponseFormat, - ) -> Result { + ) -> Result { decode_and_normalize_response(model, raw_response, request_format, normalize_response) } @@ -162,13 +167,14 @@ impl BaseOcrConfig for ReductoParseV3Config { &self, request: &PreparedOcrRequest, client: &OcrClient, - ) -> Result { - prepare_upload_request(self, request, client).await + hooks: &dyn CallHooks, + ) -> Result { + prepare_upload_request(self, request, client, hooks).await } } #[derive(Clone, Debug)] -pub(crate) struct ReductoParseLegacyConfig; +pub struct ReductoParseLegacyConfig; impl BaseOcrConfig for ReductoParseLegacyConfig { type OcrParams = ReductoLegacyParams; @@ -183,7 +189,7 @@ impl BaseOcrConfig for ReductoParseLegacyConfig { &self, non_default_params: &CallArguments, model: &str, - ) -> Result { + ) -> Result { Ok(non_default_params .select(self.get_supported_ocr_params(model)) .into()) @@ -193,7 +199,7 @@ impl BaseOcrConfig for ReductoParseLegacyConfig { &self, request: &PreparedOcrRequest, client: &OcrClient, - ) -> Result { + ) -> Result { ReductoParseV3Config .validate_environment(request, client) .await @@ -204,7 +210,7 @@ impl BaseOcrConfig for ReductoParseLegacyConfig { request: &PreparedOcrRequest, optional_params: &Self::OcrParams, environment: &Self::Environment, - ) -> Result { + ) -> Result { ReductoParseV3Config.get_complete_url(request, optional_params, environment) } @@ -214,7 +220,7 @@ impl BaseOcrConfig for ReductoParseLegacyConfig { document: OcrDocument, optional_params: &Self::OcrParams, _headers: &[(String, String)], - ) -> Result { + ) -> Result { Ok(build_legacy_body( uploaded_file_id(document)?, optional_params, @@ -228,7 +234,7 @@ impl BaseOcrConfig for ReductoParseLegacyConfig { optional_params: &ReductoLegacyParams, headers: &[(String, String)], context: OcrRequestContext<'_>, - ) -> Result { + ) -> Result { let file_id = ensure_file_id_async(document, headers, context).await?; Ok(build_legacy_body(file_id, optional_params)) } @@ -238,7 +244,7 @@ impl BaseOcrConfig for ReductoParseLegacyConfig { model: &str, raw_response: &[u8], request_format: OcrResponseFormat, - ) -> Result { + ) -> Result { ReductoParseV3Config.transform_ocr_response(model, raw_response, request_format) } @@ -246,8 +252,9 @@ impl BaseOcrConfig for ReductoParseLegacyConfig { &self, request: &PreparedOcrRequest, client: &OcrClient, - ) -> Result { - prepare_upload_request(self, request, client).await + hooks: &dyn CallHooks, + ) -> Result { + prepare_upload_request(self, request, client, hooks).await } } @@ -258,11 +265,12 @@ async fn prepare_upload_request Result { + hooks: &dyn CallHooks, +) -> Result { let params = config.map_ocr_params(&request.optional_params, &request.model)?; let headers = config.validate_environment(request, client).await?; let url = config.get_complete_url(request, ¶ms, &headers)?; - let (document, headers) = guardrail_document(request, &url, &headers).await?; + let (document, headers) = guardrail_document(request, &url, &headers, hooks).await?; let body = config .async_transform_ocr_request( &request.model, @@ -283,15 +291,15 @@ async fn prepare_upload_request Result { +fn uploaded_file_id(document: OcrDocument) -> Result { if !document.source().starts_with(REDUCTO_ID_PREFIX) { - return Err(crate::ocr::Error::ReductoSource); + return Err(Error::ReductoSource); } if document.source()[REDUCTO_ID_PREFIX.len()..] .trim() .is_empty() { - return Err(crate::ocr::Error::RequestField { + return Err(Error::RequestField { path: "document file id".into(), }); } @@ -320,10 +328,10 @@ fn checked_truncated_i64(value: f64) -> Option { .then(|| value.trunc() as i64) } -pub(crate) fn normalize_response( +pub fn normalize_response( model: &str, response: ReductoResponse, -) -> Result { +) -> Result { let result = match response.result { Some(result) => result.unwrap_or_default(), None => ReductoResult { @@ -344,7 +352,7 @@ pub(crate) fn normalize_response( }) } -fn build_pages_from_reducto(chunks: Vec) -> Result, crate::ocr::Error> { +fn build_pages_from_reducto(chunks: Vec) -> Result, Error> { let blocks_by_page = chunks .iter() .flat_map(|chunk| chunk.blocks.iter().flatten()) @@ -374,7 +382,7 @@ fn build_pages_from_reducto(chunks: Vec) -> Result, c .map(|block| match block.get("content") { None | Some(Value::Null) => Ok(None), Some(Value::String(content)) => Ok(Some(content.as_str())), - Some(_) => Err(crate::ocr::Error::ResponseField { + Some(_) => Err(Error::ResponseField { path: "result.chunks.blocks.content".into(), }), }) @@ -408,11 +416,11 @@ fn page(index: i64, markdown: String, blocks: Option) -> OcrPage { ..Default::default() } } -fn build_ocr_url(api_base: Option<&str>) -> Result { +fn build_ocr_url(api_base: Option<&str>) -> Result { complete_endpoint_url(api_base, "parse") } -fn complete_endpoint_url(api_base: Option<&str>, path: &str) -> Result { +fn complete_endpoint_url(api_base: Option<&str>, path: &str) -> Result { let base = api_base .map(str::trim) .filter(|base| !base.is_empty()) @@ -420,7 +428,7 @@ fn complete_endpoint_url(api_base: Option<&str>, path: &str) -> Result, path: &str) -> Result Option + Sync), -) -> Result, crate::ocr::Error> { - if crate::http_utils::has_header(&connection.extra_headers, "authorization") { +) -> Result, Error> { + if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization") { return Ok(connection.extra_headers.clone()); } let api_key = connection @@ -443,7 +451,7 @@ fn resolve_headers( .map(|key| key.trim().to_string()) .filter(|key| !key.is_empty()) }) - .ok_or(crate::ocr::Error::MissingReductoApiKey)?; + .ok_or(Error::MissingReductoApiKey)?; Ok( std::iter::once(("Authorization".into(), format!("Bearer {api_key}"))) .chain(connection.extra_headers.clone()) @@ -470,22 +478,21 @@ async fn ensure_file_id_async( document: OcrDocument, headers: &[(String, String)], context: OcrRequestContext<'_>, -) -> Result { +) -> Result { if document.source().starts_with(REDUCTO_ID_PREFIX) { if document.source()[REDUCTO_ID_PREFIX.len()..] .trim() .is_empty() { - return Err(crate::ocr::Error::RequestField { + return Err(Error::RequestField { path: "document file id".into(), }); } return Ok(ReductoFileId(document.source().to_string())); } - let inline = - InlineDocument::parse(document.source())?.ok_or(crate::ocr::Error::ReductoSource)?; + let inline = InlineDocument::parse(document.source())?.ok_or(Error::ReductoSource)?; let mime = inline.mime_type().to_string(); - let bytes = inline.decode(crate::constants::OCR_INLINE_MAX_BYTES)?; + let bytes = inline.decode(OCR_INLINE_MAX_BYTES)?; upload_bytes_async(bytes, &mime, headers, context).await } @@ -494,12 +501,12 @@ async fn upload_bytes_async( mime: &str, headers: &[(String, String)], context: OcrRequestContext<'_>, -) -> Result { +) -> Result { let OcrRequestContext { client, connection } = context; let part = reqwest::multipart::Part::bytes(bytes) .file_name("document") .mime_str(mime) - .map_err(|_| crate::ocr::Error::InvalidDataUri)?; + .map_err(|_| Error::InvalidDataUri)?; let builder = client .provider_http() .post(complete_endpoint_url( @@ -508,28 +515,32 @@ async fn upload_bytes_async( )?) .multipart(reqwest::multipart::Form::new().part("file", part)) .timeout(connection.timeout); - let builder = crate::http_utils::with_headers( + let builder = crate::custom_httpx::http_handler::with_headers( builder, headers, - crate::http_utils::HeaderPolicy::Except(&["content-type", "content-length"]), + crate::custom_httpx::http_handler::HeaderPolicy::Except(&[ + "content-type", + "content-length", + ]), ); - let response = crate::http_utils::http_request(builder) + let response = crate::custom_httpx::http_handler::http_request(builder) .await - .map_err(crate::transport::Error::from)?; - let uploaded = crate::ocr::client::read_json_response::( - response, - false, - connection.max_response_bytes, - ) - .await? - .data; + .map_err(crate::custom_httpx::transport::Error::from)?; + let uploaded = + crate::custom_httpx::llm_http_handler::read_json_response::( + response, + false, + connection.max_response_bytes, + ) + .await? + .data; let file_id = uploaded .file_id .as_deref() .map(str::trim) .filter(|id| !id.is_empty()); let Some(file_id) = file_id else { - return Err(crate::ocr::Error::ResponseField { + return Err(Error::ResponseField { path: "file_id".into(), }); }; @@ -590,45 +601,6 @@ mod tests { assert_eq!(normalized.usage_info.unwrap().credits, Some(1.0)); } - #[tokio::test] - async fn v3_options_preserve_explicit_null() { - let overrides = - serde_json::from_value(json!({"formatting":null,"settings":{},"unknown":true})) - .unwrap(); - let params = ReductoParseV3Config - .map_ocr_params(&overrides, "parse-v3") - .unwrap(); - let client = crate::ocr::test_support::ocr_client(); - let connection = OcrConnection::default(); - let document = serde_json::from_value( - json!({"type":"document_url","document_url":"reducto://ready.pdf"}), - ) - .unwrap(); - let body = ReductoParseV3Config - .async_transform_ocr_request( - "parse-v3", - document, - ¶ms, - &[], - OcrRequestContext { - client: &client, - connection: &connection, - }, - ) - .await - .unwrap(); - assert_eq!( - serde_json::to_value(body).unwrap(), - json!({ - "input":"reducto://ready.pdf", "formatting":null, "settings":{} - }) - ); - let absent = ReductoParseV3Config - .map_ocr_params(&crate::call_arguments::CallArguments::default(), "parse-v3") - .unwrap(); - assert_eq!(serde_json::to_value(absent).unwrap(), json!({})); - } - #[test] fn legacy_body_omits_null_enhance_and_wraps_mapped_options() { for (value, expected) in [ @@ -686,178 +658,9 @@ mod tests { ); } - use litellm_callbacks::event::{CallEvent, WireRequest}; - use rstest::rstest; - - use crate::ocr::{ - LocalOcrHost, - test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request}, - }; - - fn request_body(request: &str) -> Value { - serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap() - } - - #[rstest] - #[case( - "reducto/parse-v3", - json!({ - "formatting":{"table_output_format":"html"}, - "retrieval":{"chunk_mode":"section"}, - "settings":{"ocr_system":"standard"}, - "future_ocr_option":true, - "extra_body":{"provider_option":"value"} - }), - "reducto://already.pdf", - json!({ - "input":"reducto://already.pdf", - "formatting":{"table_output_format":"html"}, - "retrieval":{"chunk_mode":"section"}, - "settings":{"ocr_system":"standard"}, - "future_ocr_option":true, - "provider_option":"value" - }) - )] - #[case( - "reducto/parse-legacy", - json!({ - "enhance":{"agentic":[{"type":"table"}]}, - "future_ocr_option":true, - "extra_body":{"provider_option":"value"} - }), - "reducto://legacy.pdf", - json!({ - "document_url":"reducto://legacy.pdf", - "options":{"enhance":{"agentic":[{"type":"table"}]}}, - "future_ocr_option":true, - "provider_option":"value" - }) - )] - #[tokio::test] - async fn request_mapping_matches_python( - #[case] model: &str, - #[case] options: Value, - #[case] source: &str, - #[case] expected: Value, - ) { - let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ - "result":{"chunks":[]} - }))]) - .await; - let request = - crate::ocr::test_support::with_source(wire_request(model, &base, options), source); - - perform_ocr(request).await.unwrap(); - server.await.unwrap(); - let requests = seen.lock().unwrap(); - assert_eq!(requests.len(), 1); - assert!(requests[0].starts_with("POST /parse ")); - assert_eq!(request_body(&requests[0]), expected); - } - - #[rstest] - #[case("parse-v3")] - #[case("parse-legacy")] - #[tokio::test] - async fn data_uri_upload_preserves_multipart_headers(#[case] model: &str) { - let (base, seen, server) = mock_server(vec![ - MockResponse::json(json!({"file_id":"reducto://uploaded.pdf"})), - MockResponse::json(json!({"result":{"chunks":[{"content":"hello"}]}})), - ]) - .await; - let mut request = wire_request(&format!("reducto/{model}"), &base, json!({})); - request.transport.extra_headers = vec![ - ("Content-Type".into(), "application/json".into()), - ("X-Trace".into(), "upload-test".into()), - ]; - - let response = perform_ocr(request).await.unwrap(); - server.await.unwrap(); - assert_eq!(response.pages[0].markdown, "hello"); - let requests = seen.lock().unwrap(); - assert_eq!(requests.len(), 2); - assert!(requests[0].starts_with("POST /upload ")); - assert!( - requests[0] - .to_ascii_lowercase() - .contains("content-type: multipart/form-data; boundary=") - ); - assert!(requests[0].contains("x-trace: upload-test")); - assert!(requests[0].contains("application/pdf")); - assert!(requests[0].contains("abc")); - assert!(requests[1].starts_with("POST /parse ")); - } - - #[tokio::test] - async fn response_received_stays_after_reducto_upload_and_parse() { - let (base, seen, server) = mock_server(vec![ - MockResponse::json(json!({"file_id":"reducto://uploaded.pdf"})), - MockResponse::json(json!({"result":{"chunks":[]}})), - ]) - .await; - let request_count = seen.clone(); - let host = LocalOcrHost::new(wire_request("reducto/parse-v3", &base, json!({}))) - .with_observer(move |event| { - if let CallEvent::ResponseReceived { raw } = event { - assert_eq!(request_count.lock().unwrap().len(), 2); - assert_eq!(raw.body, r#"{"result":{"chunks":[]}}"#); - } - }); - - perform_ocr_with(host).await.unwrap(); - server.await.unwrap(); - assert_eq!(seen.lock().unwrap().len(), 2); - } - - #[rstest] - #[case(json!({"file_id":""}))] - #[case(json!({}))] - #[case(json!({"file_id":null}))] - #[tokio::test] - async fn invalid_upload_ids_stop_before_parse(#[case] response: Value) { - let (base, seen, server) = mock_server(vec![MockResponse::json(response)]).await; - let error = perform_ocr(wire_request("reducto/parse-v3", &base, json!({}))) - .await - .unwrap_err(); - server.await.unwrap(); - assert!(error.to_string().contains("file_id")); - assert_eq!(seen.lock().unwrap().len(), 1); - } - - #[tokio::test] - async fn upload_failure_stops_before_parse() { - let (base, seen, server) = mock_server(vec![MockResponse { - status: 503, - headers: vec![], - body: json!({"error":"unavailable"}), - }]) - .await; - assert!( - perform_ocr(wire_request("reducto/parse-v3", &base, json!({}))) - .await - .is_err() - ); - server.await.unwrap(); - assert_eq!(seen.lock().unwrap().len(), 1); - } - - #[rstest] - #[case("https://example.com/a.pdf")] - #[case("reducto://")] - #[case("data:application/pdf;base64")] - #[case("data:application/pdf;base64,INVALID!")] - #[tokio::test] - async fn rejects_invalid_document_sources_before_network(#[case] source: &str) { - let request = crate::ocr::test_support::with_source( - wire_request("reducto/parse-v3", "http://127.0.0.1:1", json!({})), - source, - ); - assert!(perform_ocr(request).await.is_err()); - } - #[test] fn response_normalization_groups_blocks_and_distinguishes_null_result() { - use crate::llms::reducto::ocr::transformation::{ReductoResponse, normalize_response}; + use crate::reducto::ocr::transformation::{ReductoResponse, normalize_response}; let raw = json!({"usage":{"num_pages":"2","credits":"3"},"result":{"type":"full","chunks":[ {"blocks":[{ @@ -901,79 +704,4 @@ mod tests { let null = normalize_response("parse-v3", null).unwrap(); assert!(null.pages.is_empty()); } - - #[tokio::test] - async fn facade_omits_native_response_by_default_and_preserves_auth_priority() { - let raw = json!({"job_id":"job-1","result":{"chunks":[]}}); - let (base, seen, server) = mock_server(vec![MockResponse::json(raw)]).await; - let mut request = crate::ocr::test_support::with_source( - wire_request("reducto/parse-v3", &base, json!({})), - "reducto://ready.pdf", - ); - request.transport.extra_headers = vec![("authorization".into(), "Bearer existing".into())]; - - let response = perform_ocr(request).await.unwrap(); - server.await.unwrap(); - assert_eq!(response.provider_native_response, None); - assert!( - seen.lock().unwrap()[0] - .to_ascii_lowercase() - .contains("authorization: bearer existing") - ); - } - - #[rstest] - #[case("reducto/parse-v3")] - #[case("reducto/parse-legacy")] - #[tokio::test] - async fn guardrail_headers_reach_upload_and_parse(#[case] model: &str) { - let (base, seen, server) = mock_server(vec![ - MockResponse::json(json!({"file_id":"reducto://uploaded.pdf"})), - MockResponse::json(json!({"result":{"chunks":[]}})), - ]) - .await; - let mut request = wire_request(model, &base, json!({})); - request.transport.extra_headers = vec![("authorization".into(), "Bearer original".into())]; - let host = LocalOcrHost::new(request).with_before_send(|wire, _| { - Ok(WireRequest { - headers: vec![("authorization".into(), "Bearer guarded".into())], - ..wire - }) - }); - - perform_ocr_with(host).await.unwrap(); - server.await.unwrap(); - let requests = seen.lock().unwrap(); - assert_eq!(requests.len(), 2); - assert!(requests[0].starts_with("POST /upload ")); - assert!(requests[1].starts_with("POST /parse ")); - for request in requests.iter() { - assert!(request.contains("authorization: Bearer guarded")); - assert!(!request.contains("Bearer original")); - } - } - - #[tokio::test] - async fn guardrail_rewrites_document_before_upload() { - let (base, seen, server) = - mock_server(vec![MockResponse::json(json!({"result":{"chunks":[]}}))]).await; - let host = LocalOcrHost::new(wire_request("reducto/parse-v3", &base, json!({}))) - .with_before_send(|wire, _| { - assert_eq!( - wire.body["document_url"], - "data:application/pdf;base64,YWJj" - ); - Ok(WireRequest { - body: json!({"type":"document_url","document_url":"reducto://guarded.pdf"}), - ..wire - }) - }); - - perform_ocr_with(host).await.unwrap(); - server.await.unwrap(); - let requests = seen.lock().unwrap(); - assert_eq!(requests.len(), 1); - assert!(requests[0].starts_with("POST /parse ")); - assert!(requests[0].contains("reducto://guarded.pdf")); - } } diff --git a/litellm-rust/crates/llms/src/vertex_ai/mod.rs b/litellm-rust/crates/llms/src/vertex_ai/mod.rs new file mode 100644 index 00000000000..3621ff6a2fd --- /dev/null +++ b/litellm-rust/crates/llms/src/vertex_ai/mod.rs @@ -0,0 +1 @@ +pub mod ocr; diff --git a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/common_utils.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs similarity index 75% rename from litellm-rust/crates/core/src/llms/vertex_ai/ocr/common_utils.rs rename to litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs index 08ffbc43cd5..979c9526f96 100644 --- a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/common_utils.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs @@ -1,8 +1,8 @@ use litellm_auth::InputSource; -use crate::ocr::types::OcrConnection; +use crate::base_llm::ocr::{error::Error, transformation::OcrConnection}; -pub(super) fn validate_destination(connection: &OcrConnection) -> Result<(), crate::ocr::Error> { +pub(super) fn validate_destination(connection: &OcrConnection) -> Result<(), Error> { if connection.api_base.is_some() && connection.api_base_source == InputSource::Request { return Err(litellm_auth::Error::RequestVertexCredentialDestination.into()); } diff --git a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/deepseek_transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs similarity index 76% rename from litellm-rust/crates/core/src/llms/vertex_ai/ocr/deepseek_transformation.rs rename to litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs index 6aece071d26..588b5243004 100644 --- a/litellm-rust/crates/core/src/llms/vertex_ai/ocr/deepseek_transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs @@ -1,21 +1,19 @@ use litellm_auth_gcp::{self as vertex, VertexConfig}; +use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl}; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use super::transformation::VertexAiOcrConfig; use crate::{ - call_arguments::CallArguments, - llms::base_llm::ocr::transformation::{BaseOcrConfig, OcrRequestContext}, - ocr::{ - OcrClient, - prepare::credential_env, - types::{ - LiteLLMOcrResponse, OcrDocument, OcrPage, OcrPageDimensions, OcrPageImage, - OcrUsageInfo, PreparedOcrRequest, + base_llm::ocr::{ + error::Error, + transformation::{ + BaseOcrConfig, LiteLLMOcrResponse, OcrDocument, OcrPage, OcrPageDimensions, + OcrPageImage, OcrRequestContext, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest, + credential_env, decode_and_normalize_response, decode_response_value, }, }, - params::OpaqueParams, - url_utils::ApiUrl, + custom_httpx::llm_http_handler::OcrClient, }; const DEFAULT_API_BASE: &str = "https://aiplatform.googleapis.com"; @@ -23,10 +21,10 @@ const MODEL_PREFIX: &str = "deepseek-ai/"; const DEFAULT_LOCATION: &str = "us-central1"; const DEEPSEEK_OCR_PARAMS: &[&str] = &["stream", "temperature", "max_tokens", "top_p", "n", "stop"]; -pub(crate) type DeepSeekOcrParams = OpaqueParams; +pub type DeepSeekOcrParams = OpaqueParams; #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct DeepSeekOcrRequest { +pub struct DeepSeekOcrRequest { pub model: String, pub messages: Vec, #[serde(flatten)] @@ -34,26 +32,26 @@ pub(crate) struct DeepSeekOcrRequest { } #[derive(Clone, Debug, Serialize, Deserialize)] -pub(crate) struct DeepSeekOcrMessage { +pub struct DeepSeekOcrMessage { pub role: UserRole, pub content: Vec, } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(tag = "type")] -pub(crate) enum DeepSeekDocument { +pub enum DeepSeekDocument { #[serde(rename = "image_url")] ImageUrl { image_url: String }, } #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] -pub(crate) enum UserRole { +pub enum UserRole { User, } #[derive(Clone, Debug, Deserialize)] -pub(crate) struct DeepSeekOcrResponse { +pub struct DeepSeekOcrResponse { #[serde(default)] choices: Vec, #[serde(default = "empty_object")] @@ -82,7 +80,7 @@ enum DeepSeekContent { #[derive(Deserialize)] struct DeepSeekPage { #[serde(default)] - #[serde_as(deserialize_as = "crate::serde_compat::LaxI64")] + #[serde_as(deserialize_as = "litellm_core_utils::serde_compat::LaxI64")] index: i64, #[serde(default)] markdown: String, @@ -91,7 +89,7 @@ struct DeepSeekPage { } #[derive(Clone, Debug)] -pub(crate) struct VertexAIDeepSeekOCRConfig; +pub struct VertexAIDeepSeekOCRConfig; impl BaseOcrConfig for VertexAIDeepSeekOCRConfig { type OcrParams = DeepSeekOcrParams; @@ -106,7 +104,7 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig { &self, _arguments: &CallArguments, _model: &str, - ) -> Result { + ) -> Result { Ok(DeepSeekOcrParams::default()) } @@ -114,7 +112,7 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig { &self, request: &PreparedOcrRequest, client: &OcrClient, - ) -> Result { + ) -> Result { VertexAiOcrConfig .validate_environment(request, client) .await @@ -125,7 +123,7 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig { request: &PreparedOcrRequest, _params: &Self::OcrParams, environment: &Self::Environment, - ) -> Result { + ) -> Result { let config = VertexConfig::from_sourced_optional_params( &request.optional_params, &request.input_sources, @@ -146,7 +144,7 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig { optional_params: &DeepSeekOcrParams, headers: &[(String, String)], _context: OcrRequestContext<'_>, - ) -> Result { + ) -> Result { self.transform_ocr_request(model, document, optional_params, headers) } @@ -154,14 +152,9 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig { &self, model: &str, raw_response: &[u8], - request_format: crate::ocr::types::OcrResponseFormat, - ) -> Result { - crate::llms::base_llm::ocr::transformation::decode_and_normalize_response( - model, - raw_response, - request_format, - normalize_response, - ) + request_format: OcrResponseFormat, + ) -> Result { + decode_and_normalize_response(model, raw_response, request_format, normalize_response) } fn transform_ocr_request( @@ -170,9 +163,9 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig { document: OcrDocument, optional_params: &DeepSeekOcrParams, _headers: &[(String, String)], - ) -> Result { + ) -> Result { if document.source().is_empty() { - return Err(crate::ocr::Error::MissingDocumentUrl); + return Err(Error::MissingDocumentUrl); } Ok(DeepSeekOcrRequest { model: provider_model(model)?, @@ -191,19 +184,19 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig { } } -pub(crate) fn normalize_response( +pub fn normalize_response( model: &str, response: DeepSeekOcrResponse, -) -> Result { +) -> Result { let content = response .choices .into_iter() .next() .and_then(|choice| choice.message.content) - .ok_or(crate::ocr::Error::EmptyContent)?; + .ok_or(Error::EmptyContent)?; let (ocr_data, fallback_markdown) = match content { DeepSeekContent::Text(text) if text.is_empty() => { - return Err(crate::ocr::Error::EmptyContent); + return Err(Error::EmptyContent); } DeepSeekContent::Text(text) => { let parsed = text @@ -214,7 +207,7 @@ pub(crate) fn normalize_response( (parsed.unwrap_or_default(), text) } DeepSeekContent::Object(data) if data.is_empty() => { - return Err(crate::ocr::Error::EmptyContent); + return Err(Error::EmptyContent); } DeepSeekContent::Object(data) => { let fallback = if data.contains_key("pages") { @@ -238,7 +231,7 @@ pub(crate) fn normalize_response( .enumerate() .filter(|(_, page)| page.is_object()) .map(|(position, page)| { - let page: DeepSeekPage = crate::ocr::json::decode_response_value( + let page: DeepSeekPage = decode_response_value( page.clone(), &format!("choices[0].message.content.pages[{position}]"), )?; @@ -250,7 +243,7 @@ pub(crate) fn normalize_response( ..Default::default() }) }) - .collect::, crate::ocr::Error>>()?, + .collect::, Error>>()?, Some(_) => return Err(response_field("pages")), None => Vec::new(), }; @@ -259,7 +252,7 @@ pub(crate) fn normalize_response( .or_else(|| (!has_pages).then_some(&response.usage)); let usage_info: Option = usage .filter(|usage| usage.is_object()) - .map(|usage| crate::ocr::json::decode_response_value(usage.clone(), "usage_info")) + .map(|usage| decode_response_value(usage.clone(), "usage_info")) .transpose()?; let model = match ocr_data.get("model") { Some(Value::String(model)) => model.clone(), @@ -359,16 +352,16 @@ impl serde_json::ser::Formatter for PythonJsonFormatter { } } -fn response_field(field: &str) -> crate::ocr::Error { - crate::ocr::Error::ResponseField { +fn response_field(field: &str) -> Error { + Error::ResponseField { path: format!("choices[0].message.content.{field}"), } } -pub(crate) fn provider_model(model: &str) -> Result { +pub fn provider_model(model: &str) -> Result { let local_model = model.trim_start_matches(MODEL_PREFIX); if local_model.is_empty() { - return Err(crate::ocr::Error::RequestField { + return Err(Error::RequestField { path: "model".into(), }); } @@ -381,7 +374,7 @@ impl VertexAIDeepSeekOCRConfig { api_base: Option<&str>, project: &str, location: &str, - ) -> Result { + ) -> Result { let base = api_base .map(str::trim) .filter(|base| !base.is_empty()) @@ -401,7 +394,7 @@ impl VertexAIDeepSeekOCRConfig { ]) }) .map(|url| url.into_string()) - .map_err(|_| crate::ocr::Error::RequestField { + .map_err(|_| Error::RequestField { path: "api_base".into(), }) } @@ -409,18 +402,24 @@ impl VertexAIDeepSeekOCRConfig { #[cfg(test)] mod tests { + use rstest::rstest; use serde_json::{Value, json}; use super::{ DeepSeekOcrParams, DeepSeekOcrResponse, VertexAIDeepSeekOCRConfig, normalize_response, provider_model, }; + use crate::base_llm::ocr::transformation::{BaseOcrConfig, OcrDocument}; + + fn document() -> OcrDocument { + serde_json::from_value(json!({"type":"image_url","image_url":"gs://bucket/a.png"})).unwrap() + } #[test] fn unconsumed_options_remain_available_for_body_composition() { use serde_json::json; - use crate::llms::base_llm::ocr::transformation::BaseOcrConfig; + use crate::base_llm::ocr::transformation::BaseOcrConfig; let arguments = serde_json::from_value(json!({"temperature":0.5,"extension":null})).unwrap(); @@ -434,8 +433,12 @@ mod tests { json!({}) ); assert_eq!( - crate::call_arguments::compose_body(&arguments, &json!({"model":"deepseek-ocr"}), &[]) - .unwrap(), + litellm_core_utils::call_arguments::compose_body( + &arguments, + &json!({"model":"deepseek-ocr"}), + &[] + ) + .unwrap(), json!({"model":"deepseek-ocr","temperature":0.5,"extension":null}) ); } @@ -458,14 +461,6 @@ mod tests { ); } - use rstest::rstest; - - use crate::{llms::base_llm::ocr::transformation::BaseOcrConfig, ocr::types::OcrDocument}; - - fn document() -> OcrDocument { - serde_json::from_value(json!({"type":"image_url","image_url":"gs://bucket/a.png"})).unwrap() - } - #[rstest] #[case("stream", json!(true))] #[case("temperature", json!(0.1))] @@ -612,93 +607,4 @@ mod tests { ); } } - - use litellm_auth::InputSource; - - use crate::ocr::test_support::{MockResponse, mock_server, perform_ocr, wire_request}; - - fn request_body(request: &str) -> Value { - serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap() - } - - #[tokio::test] - async fn facade_executes_vertex_deepseek_at_the_openai_endpoint() { - let (base, seen, server) = mock_server(vec![MockResponse::json(json!({ - "choices":[{"message":{"content":"recognized"}}], - "usage":{"prompt_tokens":1} - }))]) - .await; - let request = wire_request( - "vertex_ai/deepseek-ocr-maas", - &base, - json!({ - "vertex_project":"project-1", - "vertex_location":"europe-west4", - "temperature":0.1, - "future_ocr_option":true, - "extra_body":{"provider_option":"value"} - }), - ); - let request = crate::ocr::test_support::with_source(request, "gs://bucket/document.pdf"); - - let response = perform_ocr(request).await.unwrap(); - server.await.unwrap(); - assert_eq!(response.pages[0].markdown, "recognized"); - assert_eq!( - response.usage_info.unwrap().extra_fields["prompt_tokens"], - 1 - ); - let requests = seen.lock().unwrap(); - assert!(requests[0].starts_with( - "POST /v1/projects/project-1/locations/europe-west4/endpoints/openapi/chat/completions " - )); - assert!( - requests[0] - .to_ascii_lowercase() - .contains("authorization: bearer test-key") - ); - let body = request_body(&requests[0]); - assert_eq!(body["model"], "deepseek-ai/deepseek-ocr-maas"); - assert_eq!(body["temperature"], 0.1); - assert_eq!(body["future_ocr_option"], true); - assert_eq!(body["provider_option"], "value"); - assert!(body.get("vertex_project").is_none()); - assert!(body.get("extra_body").is_none()); - assert_eq!( - body["messages"][0]["content"][0], - json!({"type":"image_url","image_url":"gs://bucket/document.pdf"}) - ); - } - - #[test] - fn host_registration_selects_deepseek_without_affecting_mistral() { - assert!(crate::ocr::is_supported_request( - "deepseek-ocr-maas", - Some("vertex_ai") - )); - assert!(crate::ocr::is_supported_request( - "mistral-ocr-maas", - Some("vertex_ai") - )); - } - - #[tokio::test] - async fn request_controlled_api_base_is_rejected_before_vertex_auth() { - let mut request = wire_request( - "vertex_ai/deepseek-ocr-maas", - "https://caller.example", - json!({"vertex_project":"project-1"}), - ); - request.credentials.api_base = Some(litellm_auth::Sourced::new( - "https://caller.example".into(), - InputSource::Request, - )); - - let error = perform_ocr(request).await.unwrap_err(); - assert!( - error - .to_string() - .contains("request-controlled Vertex AI endpoint") - ); - } } diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/mod.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/mod.rs new file mode 100644 index 00000000000..3617ace2f7f --- /dev/null +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/mod.rs @@ -0,0 +1,3 @@ +pub mod common_utils; +pub mod deepseek_transformation; +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs new file mode 100644 index 00000000000..ea0bcf3d08c --- /dev/null +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs @@ -0,0 +1,224 @@ +use litellm_auth_gcp::{self as vertex, VertexConfig}; +use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl}; +use serde_json::Value; + +use super::common_utils::validate_destination; +use crate::{ + base_llm::ocr::{ + document::{inline_remote_document, validate_inline_document}, + error::Error, + transformation::{ + BaseOcrConfig, LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrEnvironment, + OcrRequestContext, OcrResponseFormat, PreparedOcrRequest, credential_env, + }, + }, + custom_httpx::llm_http_handler::OcrClient, + mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest}, +}; + +const DEFAULT_LOCATION: &str = "us-central1"; + +#[derive(Clone, Debug, Default)] +pub struct VertexAiOcrConfig; + +impl BaseOcrConfig for VertexAiOcrConfig { + type OcrParams = OpaqueParams; + type ProviderRequest = MistralOcrRequest; + type Environment = vertex::VertexEnvironment; + + fn get_supported_ocr_params(&self, model: &str) -> &'static [&'static str] { + MistralOcrConfig.get_supported_ocr_params(model) + } + + fn get_api_key_env_var(&self) -> Option<&'static str> { + Some("VERTEX_AI_API_KEY") + } + + fn map_ocr_params( + &self, + non_default_params: &CallArguments, + model: &str, + ) -> Result { + MistralOcrConfig.map_ocr_params(non_default_params, model) + } + + async fn validate_environment( + &self, + request: &PreparedOcrRequest, + client: &OcrClient, + ) -> Result { + let config = VertexConfig::from_sourced_optional_params( + &request.optional_params, + &request.input_sources, + )?; + self.resolve_environment(&request.connection, &config, client) + .await + } + + fn get_complete_url( + &self, + request: &PreparedOcrRequest, + _optional_params: &Self::OcrParams, + environment: &Self::Environment, + ) -> Result { + let config = VertexConfig::from_sourced_optional_params( + &request.optional_params, + &request.input_sources, + )?; + let location = vertex::get_vertex_ai_location(&config, &credential_env) + .unwrap_or_else(|| DEFAULT_LOCATION.to_string()); + self.build_ocr_url( + request.connection.api_base.as_deref(), + &environment.project_id, + &location, + &request.model, + ) + } + + fn transform_ocr_request( + &self, + model: &str, + document: OcrDocument, + optional_params: &OpaqueParams, + headers: &[(String, String)], + ) -> Result { + MistralOcrConfig.transform_ocr_request(model, document, optional_params, headers) + } + + async fn async_transform_ocr_request( + &self, + model: &str, + document: OcrDocument, + optional_params: &OpaqueParams, + headers: &[(String, String)], + context: OcrRequestContext<'_>, + ) -> Result { + let document = inline_remote_document( + context.client.document_fetcher(), + document, + context.connection, + ) + .await?; + self.transform_ocr_request(model, document, optional_params, headers) + } + + fn transform_ocr_response( + &self, + model: &str, + raw_response: &[u8], + request_format: OcrResponseFormat, + ) -> Result { + MistralOcrConfig.transform_ocr_response(model, raw_response, request_format) + } + + fn validate_request_body(&self, body: &Value) -> Result<(), Error> { + validate_inline_document(&crate::custom_httpx::llm_http_handler::body_document(body)?) + } +} + +impl OcrEnvironment for vertex::VertexEnvironment { + fn headers(&self) -> &[(String, String)] { + &self.headers + } +} + +impl VertexAiOcrConfig { + async fn resolve_environment( + &self, + connection: &OcrConnection, + config: &VertexConfig, + client: &OcrClient, + ) -> Result { + validate_destination(connection)?; + client + .vertex_auth() + .validate_environment( + connection.extra_headers.clone(), + connection.api_key.as_deref(), + config, + &credential_env, + ) + .await + .map_err(Error::from) + } + + fn build_ocr_url( + &self, + api_base: Option<&str>, + project: &str, + location: &str, + model: &str, + ) -> Result { + validate_location(location)?; + let default_base = format!("https://{location}-aiplatform.googleapis.com"); + let base = api_base + .map(str::trim) + .filter(|base| !base.is_empty()) + .unwrap_or(&default_base); + let prediction = format!("{model}:rawPredict"); + ApiUrl::parse(base) + .and_then(|url| { + url.complete_path(&[ + "v1", + "projects", + project, + "locations", + location, + "publishers", + "mistralai", + "models", + &prediction, + ]) + }) + .map(|url| url.into_string()) + .map_err(|_| Error::RequestField { + path: "api_base".into(), + }) + } +} + +fn validate_location(location: &str) -> Result<(), Error> { + let valid = !location.is_empty() + && location + .bytes() + .all(|value| value.is_ascii_lowercase() || value.is_ascii_digit() || value == b'-') + && location + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && location + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric); + if valid { + return Ok(()); + } + Err(Error::RequestField { + path: "vertex_location".into(), + }) +} + +#[cfg(test)] +mod tests { + + use super::VertexAiOcrConfig; + + #[test] + fn endpoint_uses_location_project_and_model() { + assert_eq!( + VertexAiOcrConfig + .build_ocr_url(None, "proj-1", "europe-west4", "mistral-ocr-maas") + .unwrap(), + "https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict" + ); + } + + #[test] + fn endpoint_rejects_invalid_location() { + assert!( + VertexAiOcrConfig + .build_ocr_url(None, "proj-1", "attacker.example/path", "model") + .is_err() + ); + } +} diff --git a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/mod.rs b/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/mod.rs deleted file mode 100644 index ba63992f3cb..00000000000 --- a/litellm-rust/crates/providers/src/anthropic/experimental_pass_through/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod messages; diff --git a/litellm-rust/crates/providers/src/audio_transcription/mod.rs b/litellm-rust/crates/providers/src/audio_transcription/mod.rs deleted file mode 100644 index 278b049e8f9..00000000000 --- a/litellm-rust/crates/providers/src/audio_transcription/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -use thiserror::Error; - -#[derive(Clone, Debug, PartialEq, Eq, Error)] -pub enum Error { - #[error("expected {expected}, got {actual}")] - InvalidType { - expected: &'static str, - actual: &'static str, - }, - #[error("missing required field: {0}")] - MissingField(&'static str), - #[error("invalid request: {0}")] - InvalidRequest(String), - #[error("invalid response: {0}")] - InvalidResponse(String), - #[error(transparent)] - Auth(#[from] litellm_auth::Error), -} - -pub fn json_type_name(value: &serde_json::Value) -> &'static str { - match value { - serde_json::Value::Null => "null", - serde_json::Value::Bool(_) => "boolean", - serde_json::Value::Number(_) => "number", - serde_json::Value::String(_) => "string", - serde_json::Value::Array(_) => "array", - serde_json::Value::Object(_) => "object", - } -} - -pub mod types; diff --git a/litellm-rust/crates/providers/src/chat/mod.rs b/litellm-rust/crates/providers/src/chat/mod.rs deleted file mode 100644 index 93892657c75..00000000000 --- a/litellm-rust/crates/providers/src/chat/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -use thiserror::Error; - -pub const EMPTY_TEXT_PLACEHOLDER: &str = " "; - -#[derive(Clone, Debug, PartialEq, Eq, Error)] -pub enum Error { - #[error("missing required field: {0}")] - MissingField(&'static str), - #[error("invalid request: {0}")] - InvalidRequest(String), - #[error("invalid response: {0}")] - InvalidResponse(String), - #[error("unsupported: {0}")] - Unsupported(&'static str), - #[error(transparent)] - Auth(#[from] litellm_auth::Error), -} - -pub mod conversation; -pub mod response_utils; -pub mod types; diff --git a/litellm-rust/crates/providers/src/chat/types.rs b/litellm-rust/crates/providers/src/chat/types.rs deleted file mode 100644 index d61892624cf..00000000000 --- a/litellm-rust/crates/providers/src/chat/types.rs +++ /dev/null @@ -1,202 +0,0 @@ -use std::time::Duration; - -use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; - -use crate::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth}; - -/// A `/chat/completions` call as it crosses into the core. -/// -/// `optional_params` arrives already mapped to the provider's own parameter -/// names by the host, exactly as the messages route receives an already -/// Anthropic-shaped body. The core owns the conversation translation, the -/// provider call, and the response normalization. -pub struct ChatCompletionsRequest<'a> { - pub model: &'a str, - pub messages: Value, - pub optional_params: Map, - pub api_key: Option<&'a str>, - pub api_base: Option<&'a str>, - pub custom_llm_provider: Option<&'a str>, - pub extra_headers: Option>, - pub timeout: Option, -} - -pub struct ResolvedChatCompletionsRequest<'a> { - pub model: String, - pub config: &'static dyn BaseConfig, - pub messages: Vec, - pub optional_params: Map, - pub api_key: Option<&'a str>, - pub api_base: Option<&'a str>, - pub extra_headers: Option>, - pub timeout: Option, -} - -pub struct ProviderChatCompletionsRequest { - pub model: String, - pub config: &'static dyn BaseConfig, - pub url: String, - pub body: Value, - pub upstream_headers: Vec<(String, String)>, - pub auth: ChatCompletionsAuth, - pub optional_params: Map, - pub timeout: Option, -} - -/// The provider-shaped request body a config produces. Named rather than a bare -/// `Value` so the transform contract stays a typed one, mirroring -/// [`crate::audio_transcription::types::AudioTranscriptionRequestData`]. -pub struct ProviderChatRequestData { - pub body: Value, -} - -/// The raw provider response body handed back to a config for normalization. -pub struct ProviderChatResponseData { - pub body: Value, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(untagged)] -pub enum ChatMessageContent { - Text(String), - Parts(Vec), -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatMessage { - pub role: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub content: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, - #[serde(flatten)] - pub extra: Map, -} - -/// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python -/// path reports so cost tracking sees the same numbers on either path. -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct PromptTokensDetails { - pub cached_tokens: u64, - pub cache_creation_tokens: u64, - pub text_tokens: u64, -} - -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionsUsage { - pub prompt_tokens: u64, - pub completion_tokens: u64, - pub total_tokens: u64, - pub prompt_tokens_details: PromptTokensDetails, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionsChoiceMessage { - pub role: String, - // Whether an empty turn is `None` or `""` is the provider's choice, not a - // shared invariant: Anthropic's transform ends on `merged_text or None` - // while Converse assigns the joined string unconditionally. Each config - // mirrors its own, so keep this optional and serialize it even when None. - pub content: Option, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionsChoice { - pub index: u64, - pub message: ChatCompletionsChoiceMessage, - pub finish_reason: String, -} - -/// The normalized response handed back to the host. -/// -/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the -/// `ModelResponse` it already created, and echoing the provider's own id here -/// would change it. Pinned by `response_carries_no_id` in `tests.rs`. -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionsResponse { - pub created: u64, - pub model: String, - pub choices: Vec, - pub usage: ChatCompletionsUsage, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionToolCallFunctionChunk { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub name: Option, - pub arguments: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provider_specific_fields: Option>, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionToolCallChunk { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub id: Option, - #[serde(rename = "type")] - pub tool_type: String, - pub function: ChatCompletionToolCallFunctionChunk, - pub index: i64, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum ChatCompletionThinkingBlock { - Thinking { - #[serde(default, skip_serializing_if = "Option::is_none")] - thinking: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - signature: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - cache_control: Option, - }, - RedactedThinking { - #[serde(default, skip_serializing_if = "Option::is_none")] - data: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - cache_control: Option, - }, -} - -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionDelta { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub content: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub role: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tool_calls: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reasoning_content: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub thinking_blocks: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provider_specific_fields: Option>, - #[serde(flatten)] - pub extra: Map, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionStreamingChoice { - pub index: u64, - pub delta: ChatCompletionDelta, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub finish_reason: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub logprobs: Option, -} - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct ChatCompletionChunk { - pub id: String, - pub created: u64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub model: Option, - pub object: String, - pub choices: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub usage: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub provider_specific_fields: Option>, -} diff --git a/litellm-rust/crates/providers/src/lib.rs b/litellm-rust/crates/providers/src/lib.rs deleted file mode 100644 index 5d72ffffb2b..00000000000 --- a/litellm-rust/crates/providers/src/lib.rs +++ /dev/null @@ -1,8 +0,0 @@ -pub mod anthropic; -pub mod audio_transcription; -pub mod azure_ai; -pub mod base_llm; -pub mod bedrock; -pub mod chat; -pub mod messages; -pub mod provider_resolution; diff --git a/litellm-rust/crates/providers/src/messages/mod.rs b/litellm-rust/crates/providers/src/messages/mod.rs deleted file mode 100644 index 07232b36b51..00000000000 --- a/litellm-rust/crates/providers/src/messages/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -use thiserror::Error; - -#[derive(Clone, Debug, PartialEq, Eq, Error)] -pub enum Error { - #[error("missing required field: {0}")] - MissingField(&'static str), - #[error("invalid request: {0}")] - InvalidRequest(String), - #[error("invalid response: {0}")] - InvalidResponse(String), - #[error("unsupported: {0}")] - Unsupported(&'static str), - #[error(transparent)] - Auth(#[from] litellm_auth::Error), -} - -pub mod types; diff --git a/litellm-rust/crates/providers/src/provider_resolution.rs b/litellm-rust/crates/providers/src/provider_resolution.rs deleted file mode 100644 index d1ada2472e9..00000000000 --- a/litellm-rust/crates/providers/src/provider_resolution.rs +++ /dev/null @@ -1,33 +0,0 @@ -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct CustomLlmProvider<'a> { - pub model: &'a str, - pub custom_llm_provider: &'a str, -} - -pub fn get_custom_llm_provider<'a>( - model: &'a str, - custom_llm_provider: Option<&'a str>, -) -> Option> { - if let Some(custom_llm_provider) = custom_llm_provider.filter(|provider| !provider.is_empty()) { - return Some(CustomLlmProvider { - model: strip_custom_llm_provider_prefix(model, custom_llm_provider), - custom_llm_provider, - }); - } - - let (custom_llm_provider, model) = model.split_once('/')?; - if custom_llm_provider.is_empty() || model.is_empty() { - return None; - } - Some(CustomLlmProvider { - model, - custom_llm_provider, - }) -} - -fn strip_custom_llm_provider_prefix<'a>(model: &'a str, custom_llm_provider: &str) -> &'a str { - model - .strip_prefix(custom_llm_provider) - .and_then(|model| model.strip_prefix('/')) - .unwrap_or(model) -} diff --git a/litellm-rust/crates/python-bridge/Cargo.toml b/litellm-rust/crates/python-bridge/Cargo.toml index 2959fac1084..e9b7f384406 100644 --- a/litellm-rust/crates/python-bridge/Cargo.toml +++ b/litellm-rust/crates/python-bridge/Cargo.toml @@ -20,6 +20,8 @@ bytes.workspace = true litellm-auth.workspace = true litellm-callbacks-legacy.workspace = true litellm-core.workspace = true +litellm-llms.workspace = true +litellm-types.workspace = true litellm-host-python.workspace = true litellm-token-counter.workspace = true pyo3.workspace = true diff --git a/litellm-rust/crates/python-bridge/benches/serialization.rs b/litellm-rust/crates/python-bridge/benches/serialization.rs index 7641f35932a..d398d9fdbfc 100644 --- a/litellm-rust/crates/python-bridge/benches/serialization.rs +++ b/litellm-rust/crates/python-bridge/benches/serialization.rs @@ -1,10 +1,8 @@ -use std::hint::black_box; -use std::time::Duration; +use std::{hint::black_box, time::Duration}; use criterion::{BenchmarkId, Criterion, criterion_group, criterion_main}; use litellm_host_python::{from_py, to_py}; -use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::{prelude::*, types::PyDict}; use serde_json::{Value, json}; const PAYLOAD_SIZES: &[(&str, usize)] = &[ diff --git a/litellm-rust/crates/python-bridge/src/credentials.rs b/litellm-rust/crates/python-bridge/src/credentials.rs index 5a546f9628e..44437ec2a02 100644 --- a/litellm-rust/crates/python-bridge/src/credentials.rs +++ b/litellm-rust/crates/python-bridge/src/credentials.rs @@ -3,10 +3,12 @@ use litellm_auth::{ResolvedCredential, SecretValue}; use litellm_host_python::wrap_failure; -use pyo3::exceptions::PyTypeError; -use pyo3::gc::{PyTraverseError, PyVisit}; -use pyo3::prelude::*; -use pyo3::types::{PyDict, PyString}; +use pyo3::{ + exceptions::PyTypeError, + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::{PyDict, PyString}, +}; const NOT_CALLABLE: &str = "Azure AD token provider must be callable"; const NOT_A_STRING: &str = "Azure AD token must be a string, got {}"; diff --git a/litellm-rust/crates/python-bridge/src/diagnostics.rs b/litellm-rust/crates/python-bridge/src/diagnostics.rs index 42db4510faa..39fa8bc3596 100644 --- a/litellm-rust/crates/python-bridge/src/diagnostics.rs +++ b/litellm-rust/crates/python-bridge/src/diagnostics.rs @@ -1,6 +1,5 @@ use litellm_host_python::release_count; -use pyo3::prelude::*; -use pyo3::types::PyDict; +use pyo3::{prelude::*, types::PyDict}; #[pyfunction] pub(crate) fn gil_stats(py: Python<'_>) -> PyResult> { diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index 3d6f4e2a0dd..61c5947ed9e 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -1,7 +1,11 @@ -use litellm_core::transport::Error as TransportError; -use litellm_core::{Error, audio_transcription, chat_completions, messages, ocr, responses}; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::*; +use litellm_core::{Error, audio_transcription, chat_completions, messages, responses}; +use litellm_llms::{ + base_llm::ocr::error::Error as OcrError, custom_httpx::transport::Error as TransportError, +}; +use pyo3::{ + exceptions::{PyRuntimeError, PyValueError}, + prelude::*, +}; pyo3::create_exception!( _native, @@ -39,11 +43,11 @@ pub(crate) fn core_error_to_pyerr(error: Error) -> PyErr { error.is_request() || matches!( error, - ocr::Error::Auth(_) - | ocr::Error::InvalidProvider(_) - | ocr::Error::InvalidRequest(_) - | ocr::Error::MissingField(_) - | ocr::Error::MissingDocumentUrl + OcrError::Auth(_) + | OcrError::InvalidProvider(_) + | OcrError::InvalidRequest(_) + | OcrError::MissingField(_) + | OcrError::MissingDocumentUrl ) } Error::Messages(error) => match error { diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index 294c439e7e9..ea4077b102f 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -1,13 +1,12 @@ -use std::collections::{BTreeMap, HashMap}; -use std::time::Duration; - -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::PyDict; -use serde_json::{Map, Value}; +use std::{ + collections::{BTreeMap, HashMap}, + time::Duration, +}; use litellm_auth::InputSource; use litellm_host_python::{from_py, from_py_argument}; +use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; +use serde_json::{Map, Value}; /// The keyword arguments every value route shares, validated at the Python boundary. pub(crate) struct RouteOptions { @@ -156,10 +155,11 @@ pub(crate) fn marshal_headers(headers: Option) -> PyResult(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> { let locals = PyDict::new(py); py.run(source, Some(&locals), Some(&locals)).unwrap(); diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs index 248475b26ed..d63e9a1feaf 100644 --- a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs @@ -1,13 +1,13 @@ use litellm_core::audio_transcription::{ - AudioTranscriptionRequest, Error, audio_transcription as run_audio_transcription, + Error, audio_transcription as run_audio_transcription, types::AudioTranscriptionRequest, }; use litellm_host_python::{from_py_argument, run_async, run_sync}; use pyo3::prelude::*; use serde_json::{Map, Value}; -use crate::errors::audio_transcription_error_to_pyerr; -use crate::marshal::{ - RouteOptions, extra_headers_argument, optional_params_argument, optional_timeout, +use crate::{ + errors::audio_transcription_error_to_pyerr, + marshal::{RouteOptions, extra_headers_argument, optional_params_argument, optional_timeout}, }; async fn execute( diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs index 67036c307e2..049a507dcdc 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs @@ -1,16 +1,18 @@ -use litellm_core::chat_completions::Error; -use litellm_core::chat_completions::types::{ChatCompletionsRequest, ChatCompletionsResponse}; use litellm_core::chat_completions::{ - chat_completions as run_chat_completions, chat_completions_decline_reason, + Error, chat_completions as run_chat_completions, chat_completions_decline_reason, + types::ChatCompletionsRequest, }; use litellm_host_python::{from_py_argument, run_async, run_sync}; +use litellm_types::utils::ChatCompletionsResponse; use pyo3::prelude::*; use serde_json::{Map, Value}; -use crate::errors::chat_completions_error_to_pyerr; -use crate::marshal::{ - RouteOptions, extra_headers_argument, messages_argument, optional_params_argument, - optional_timeout, +use crate::{ + errors::chat_completions_error_to_pyerr, + marshal::{ + RouteOptions, extra_headers_argument, messages_argument, optional_params_argument, + optional_timeout, + }, }; async fn execute( @@ -122,8 +124,7 @@ pub(crate) fn achat_completions<'py>( #[cfg(test)] mod tests { - use pyo3::prelude::*; - use pyo3::types::PyList; + use pyo3::{prelude::*, types::PyList}; #[test] fn chat_completions_decline_keeps_existing_reasons() { diff --git a/litellm-rust/crates/python-bridge/src/routes/messages.rs b/litellm-rust/crates/python-bridge/src/routes/messages.rs index 371e8c27171..daec931c92e 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages.rs @@ -1,12 +1,13 @@ -use litellm_core::messages::Error; -use litellm_core::messages::messages as run_messages; -use litellm_core::messages::types::{AnthropicMessagesResponse, MessagesRequest}; +use litellm_core::messages::{Error, messages as run_messages, types::MessagesRequest}; use litellm_host_python::{run_async, run_sync}; +use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; use pyo3::prelude::*; use serde_json::{Map, Value}; -use crate::errors::messages_error_to_pyerr; -use crate::marshal::{RouteOptions, body_argument, extra_headers_argument, optional_timeout}; +use crate::{ + errors::messages_error_to_pyerr, + marshal::{RouteOptions, body_argument, extra_headers_argument, optional_timeout}, +}; async fn execute( body: Map, diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index b6ada947597..f59e32a28e2 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -6,8 +6,10 @@ pub(crate) mod responses; #[cfg(test)] mod tests { - use pyo3::prelude::*; - use pyo3::types::{PyDict, PyList}; + use pyo3::{ + prelude::*, + types::{PyDict, PyList}, + }; #[test] fn sync_and_async_route_signatures_match_the_python_contract() { diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs index 1a111ca2c11..ed840dec70c 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/document.rs @@ -1,13 +1,14 @@ use std::path::PathBuf; use bytes::Bytes; -use pyo3::exceptions::{PyTypeError, PyValueError}; -use pyo3::gc::{PyTraverseError, PyVisit}; -use pyo3::prelude::*; -use pyo3::pybacked::PyBackedBytes; -use pyo3::types::{PyBytes, PyString}; - -use litellm_core::ocr::{OcrDocumentInput, OcrFileContent}; +use litellm_core::ocr::types::{OcrDocumentInput, OcrFileContent}; +use pyo3::{ + exceptions::{PyTypeError, PyValueError}, + gc::{PyTraverseError, PyVisit}, + prelude::*, + pybacked::PyBackedBytes, + types::{PyBytes, PyString}, +}; #[derive(Debug)] pub(super) struct PythonFileReader { @@ -128,9 +129,10 @@ impl FromPyObject<'_, '_> for FileDocumentInput { #[cfg(test)] mod tests { - use super::*; use pyo3::types::PyDict; + use super::*; + fn eval<'py>(py: Python<'py>, source: &std::ffi::CStr) -> Bound<'py, PyDict> { let locals = PyDict::new(py); py.run(source, Some(&locals), Some(&locals)).unwrap(); diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs index 215060b7a9b..0ae56efbf02 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs @@ -1,6 +1,8 @@ -use litellm_core::ocr::Error; -use pyo3::exceptions::{PyFileNotFoundError, PyOSError}; -use pyo3::prelude::*; +use litellm_llms::base_llm::ocr::error::Error; +use pyo3::{ + exceptions::{PyFileNotFoundError, PyOSError}, + prelude::*, +}; use crate::errors::{RustUpstreamError, core_error_to_pyerr}; @@ -13,9 +15,10 @@ pub(super) fn to_pyerr(error: Error) -> PyErr { body, headers, } => upstream_error(py, status, body, headers)?, - Error::Transport(litellm_core::transport::Error::Http { status, body }) => { - upstream_error(py, status, body, Vec::new())? - } + Error::Transport(litellm_llms::custom_httpx::transport::Error::Http { + status, + body, + }) => upstream_error(py, status, body, Vec::new())?, Error::RequestFormat => { let error = core_error_to_pyerr(Error::RequestFormat.into()); error @@ -59,9 +62,10 @@ fn attach_status(error: PyErr, status: Option) -> PyErr { #[cfg(test)] mod tests { - use super::*; use pyo3::exceptions::PyValueError; + use super::*; + #[test] fn preserves_python_validation_and_provider_details() { Python::initialize(); diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs index a0f2714753d..9dc891a91d6 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs @@ -1,13 +1,18 @@ use litellm_auth::ResolvedCredential; -use litellm_core::ocr::{LiteLLMOcrResponse, Ocr, OcrOp, OcrOpResult}; +use litellm_core::ocr::route::{Ocr, OcrOp, OcrOpResult}; use litellm_host_python::{RouteHost, missing_state, to_py}; -use pyo3::exceptions::PyBaseException; -use pyo3::gc::{PyTraverseError, PyVisit}; -use pyo3::prelude::*; -use pyo3::types::PyDict; +use litellm_llms::base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse}; +use pyo3::{ + exceptions::PyBaseException, + gc::{PyTraverseError, PyVisit}, + prelude::*, + types::PyDict, +}; -use super::errors::to_pyerr as ocr_error_to_pyerr; -use super::project::{OcrHostHandles, project_request}; +use super::{ + errors::to_pyerr as ocr_error_to_pyerr, + project::{OcrHostHandles, project_request}, +}; enum OcrHostData { Unprojected, @@ -37,7 +42,7 @@ impl OcrRouteHost { } } - fn read_document(&self, py: Python<'_>) -> PyResult { + fn read_document(&self, py: Python<'_>) -> PyResult { self.handles()? .reader .as_ref() @@ -90,12 +95,12 @@ impl RouteHost for OcrRouteHost { .map(Bound::unbind) } - fn native_error(error: litellm_core::ocr::Error) -> PyErr { + fn native_error(error: Error) -> PyErr { ocr_error_to_pyerr(error) } - fn host_error(error: &PyErr) -> litellm_core::ocr::Error { - litellm_core::ocr::Error::InvalidRequest(error.to_string()) + fn host_error(error: &PyErr) -> Error { + Error::InvalidRequest(error.to_string()) } fn map_failure(&self, py: Python<'_>, error: &PyErr) -> PyResult { diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index 87590b52dd5..b5bb941708d 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -3,12 +3,14 @@ mod errors; mod host; mod project; -use litellm_callbacks_legacy::{LegacySurface, PublicCall, run_legacy_call}; -use litellm_core::ocr::{OcrClient, ocr_machine}; -use pyo3::prelude::*; -use pyo3::types::{PyDict, PyTuple}; - use host::OcrRouteHost; +use litellm_callbacks_legacy::{LegacySurface, PublicCall, run_legacy_call}; +use litellm_core::ocr::route::ocr_machine; +use litellm_llms::custom_httpx::llm_http_handler::OcrClient; +use pyo3::{ + prelude::*, + types::{PyDict, PyTuple}, +}; const SURFACE: LegacySurface = LegacySurface { call_type: "ocr", diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index 314bdec0e1b..7ffa129f85c 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -1,17 +1,20 @@ -use litellm_core::ocr::wire::{ - OcrWireRequest, consumed_optional_params, decode_document, decode_request_input, +use litellm_core::ocr::{ + types::{LiteLLMOcrRequest, OcrDocumentInput}, + wire::{OcrWireRequest, consumed_optional_params, decode_document, decode_request_input}, }; -use litellm_core::ocr::{LiteLLMOcrRequest, OcrDocumentInput}; use litellm_host_python::from_py; -use pyo3::exceptions::PyValueError; -use pyo3::prelude::*; -use pyo3::types::PyDict; +use litellm_llms::base_llm::ocr::error::Error; +use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; use serde_json::{Map, Value}; -use super::document::{FileDocumentInput, PythonFileReader}; -use super::errors::to_pyerr as ocr_error_to_pyerr; -use crate::credentials::{self, CallerTokenProvider}; -use crate::marshal::{project_optional_fields, python_timeout_seconds, request_input_sources}; +use super::{ + document::{FileDocumentInput, PythonFileReader}, + errors::to_pyerr as ocr_error_to_pyerr, +}; +use crate::{ + credentials::{self, CallerTokenProvider}, + marshal::{project_optional_fields, python_timeout_seconds, request_input_sources}, +}; /// What the host keeps after projection: the caller's callables that answer the document /// read and token operations, and the provider name the failure mapping reports. @@ -84,7 +87,7 @@ impl ProjectedDocument { if error.is_instance_of::(py) || error.is_instance_of::(py) { - ocr_error_to_pyerr(litellm_core::ocr::Error::RequestField { + ocr_error_to_pyerr(Error::RequestField { path: "document.type".into(), }) } else { @@ -155,6 +158,7 @@ pub(super) fn project_request( #[cfg(test)] mod tests { + use litellm_llms::base_llm::ocr::transformation::OcrDocument; use pyo3::exceptions::PyValueError; use super::*; @@ -179,7 +183,7 @@ mod tests { } fn url_document(url: &str) -> OcrDocumentInput { - litellm_core::ocr::OcrDocument::DocumentUrl { + OcrDocument::DocumentUrl { document_url: url.into(), extra_fields: Default::default(), } diff --git a/litellm-rust/crates/python-bridge/src/routes/responses.rs b/litellm-rust/crates/python-bridge/src/routes/responses.rs index bf48e4619a9..9c10d58de4f 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses.rs @@ -2,8 +2,10 @@ use litellm_core::responses::websocket::ResponsesWebSocketConnection as RustResp use pyo3::prelude::*; use serde_json::Value; -use crate::errors::responses_error_to_pyerr; -use crate::marshal::{marshal_headers, optional_timeout}; +use crate::{ + errors::responses_error_to_pyerr, + marshal::{marshal_headers, optional_timeout}, +}; #[pyclass] pub(crate) struct ResponsesWebSocketConnection { @@ -58,12 +60,10 @@ impl ResponsesWebSocketConnection { #[cfg(test)] mod tests { - use std::ffi::CString; - use std::time::Duration; + use std::{ffi::CString, time::Duration}; use futures_util::{SinkExt, StreamExt}; - use pyo3::prelude::*; - use pyo3::types::PyDict; + use pyo3::{prelude::*, types::PyDict}; use tokio::net::TcpListener; use tokio_tungstenite::{accept_async, tungstenite::Message}; diff --git a/litellm-rust/crates/python-bridge/src/token_counter.rs b/litellm-rust/crates/python-bridge/src/token_counter.rs index 117e2b6e6ff..7dc86b78ad6 100644 --- a/litellm-rust/crates/python-bridge/src/token_counter.rs +++ b/litellm-rust/crates/python-bridge/src/token_counter.rs @@ -1,18 +1,17 @@ -use std::num::NonZero; -use std::sync::Arc; -use std::thread::available_parallelism; +use std::{num::NonZero, sync::Arc, thread::available_parallelism}; -use litellm_host_python::release_gil; +use litellm_host_python::{release_gil, run_async}; use litellm_token_counter::{ CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter, }; -use pyo3::exceptions::{PyRuntimeError, PyValueError}; -use pyo3::prelude::*; -use pyo3::types::PyAny; +use pyo3::{ + exceptions::{PyRuntimeError, PyValueError}, + prelude::*, + types::PyAny, +}; use tokio::sync::Semaphore; use crate::errors::RustBridgeDeclined; -use litellm_host_python::run_async; /// Counts the input tokens of a raw request body off the Python event loop with /// the GIL released. Python owns which requests get here and what to do with diff --git a/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs b/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs index e99c01ae57e..86809ecded9 100644 --- a/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs +++ b/litellm-rust/crates/python-bridge/tests/marshal_boundary.rs @@ -1,5 +1,7 @@ -use std::fs; -use std::path::{Path, PathBuf}; +use std::{ + fs, + path::{Path, PathBuf}, +}; const DISALLOWED_OUTSIDE_INTEROP: &[&str] = &[ "py.import(\"json\")", diff --git a/litellm-rust/crates/types/Cargo.toml b/litellm-rust/crates/types/Cargo.toml new file mode 100644 index 00000000000..6a2efa90ab4 --- /dev/null +++ b/litellm-rust/crates/types/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "litellm-types" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository.workspace = true + +[dependencies] +serde.workspace = true +serde_json.workspace = true diff --git a/litellm-rust/crates/types/src/lib.rs b/litellm-rust/crates/types/src/lib.rs new file mode 100644 index 00000000000..da5c9ea893f --- /dev/null +++ b/litellm-rust/crates/types/src/lib.rs @@ -0,0 +1,3 @@ +pub mod llms; +pub mod responses; +pub mod utils; diff --git a/litellm-rust/crates/providers/src/messages/types.rs b/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_request.rs similarity index 69% rename from litellm-rust/crates/providers/src/messages/types.rs rename to litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_request.rs index ba274ab9651..50eedf7ba09 100644 --- a/litellm-rust/crates/providers/src/messages/types.rs +++ b/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_request.rs @@ -1,30 +1,6 @@ -use std::time::Duration; - use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; -use crate::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig; - -pub struct MessagesRequest<'a> { - pub model: &'a str, - pub body: Value, - pub api_key: Option<&'a str>, - pub api_base: Option<&'a str>, - pub custom_llm_provider: Option<&'a str>, - pub extra_headers: Option>, - pub timeout: Option, -} - -pub struct ProviderMessagesRequest { - pub provider: String, - pub model: String, - pub config: &'static dyn BaseAnthropicMessagesConfig, - pub url: String, - pub body: Value, - pub upstream_headers: Vec<(String, String)>, - pub timeout: Option, -} - #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] #[serde(untagged)] pub enum SystemPrompt { @@ -112,23 +88,3 @@ pub struct AnthropicMessagesRequest { #[serde(flatten)] pub extra: Map, } - -#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] -pub struct AnthropicMessagesResponse { - pub id: String, - #[serde(rename = "type")] - pub message_type: String, - pub role: String, - pub model: String, - pub content: Vec, - // Anthropic always includes stop_reason / stop_sequence, null until the turn - // ends; serialize them even when None so callers see the same shape as Python. - pub stop_reason: Option, - pub stop_sequence: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub usage: Option, - #[serde(skip_serializing_if = "Option::is_none")] - pub container: Option, - #[serde(flatten)] - pub extra: Map, -} diff --git a/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_response.rs b/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_response.rs new file mode 100644 index 00000000000..0c3876aac59 --- /dev/null +++ b/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_response.rs @@ -0,0 +1,22 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct AnthropicMessagesResponse { + pub id: String, + #[serde(rename = "type")] + pub message_type: String, + pub role: String, + pub model: String, + pub content: Vec, + // Anthropic always includes stop_reason / stop_sequence, null until the turn + // ends; serialize them even when None so callers see the same shape as Python. + pub stop_reason: Option, + pub stop_sequence: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub container: Option, + #[serde(flatten)] + pub extra: Map, +} diff --git a/litellm-rust/crates/types/src/llms/anthropic_messages/mod.rs b/litellm-rust/crates/types/src/llms/anthropic_messages/mod.rs new file mode 100644 index 00000000000..2b6ada1f22e --- /dev/null +++ b/litellm-rust/crates/types/src/llms/anthropic_messages/mod.rs @@ -0,0 +1,2 @@ +pub mod anthropic_request; +pub mod anthropic_response; diff --git a/litellm-rust/crates/types/src/llms/mod.rs b/litellm-rust/crates/types/src/llms/mod.rs new file mode 100644 index 00000000000..09d2207a0ca --- /dev/null +++ b/litellm-rust/crates/types/src/llms/mod.rs @@ -0,0 +1,2 @@ +pub mod anthropic_messages; +pub mod openai; diff --git a/litellm-rust/crates/types/src/llms/openai.rs b/litellm-rust/crates/types/src/llms/openai.rs new file mode 100644 index 00000000000..232f5b9cc51 --- /dev/null +++ b/litellm-rust/crates/types/src/llms/openai.rs @@ -0,0 +1,58 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ChatMessageContent { + Text(String), + Parts(Vec), +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatMessage { + pub role: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionToolCallFunctionChunk { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub arguments: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option>, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionToolCallChunk { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(rename = "type")] + pub tool_type: String, + pub function: ChatCompletionToolCallFunctionChunk, + pub index: i64, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ChatCompletionThinkingBlock { + Thinking { + #[serde(default, skip_serializing_if = "Option::is_none")] + thinking: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + signature: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cache_control: Option, + }, + RedactedThinking { + #[serde(default, skip_serializing_if = "Option::is_none")] + data: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + cache_control: Option, + }, +} diff --git a/litellm-rust/crates/types/src/responses/mod.rs b/litellm-rust/crates/types/src/responses/mod.rs new file mode 100644 index 00000000000..02493c5f6ed --- /dev/null +++ b/litellm-rust/crates/types/src/responses/mod.rs @@ -0,0 +1 @@ +pub mod streaming_websocket; diff --git a/litellm-rust/crates/core/src/responses/types.rs b/litellm-rust/crates/types/src/responses/streaming_websocket.rs similarity index 100% rename from litellm-rust/crates/core/src/responses/types.rs rename to litellm-rust/crates/types/src/responses/streaming_websocket.rs diff --git a/litellm-rust/crates/types/src/utils.rs b/litellm-rust/crates/types/src/utils.rs new file mode 100644 index 00000000000..7f0c18f9f2c --- /dev/null +++ b/litellm-rust/crates/types/src/utils.rs @@ -0,0 +1,93 @@ +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +use crate::llms::openai::{ChatCompletionThinkingBlock, ChatCompletionToolCallChunk}; + +/// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python +/// path reports so cost tracking sees the same numbers on either path. +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct PromptTokensDetails { + pub cached_tokens: u64, + pub cache_creation_tokens: u64, + pub text_tokens: u64, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionsUsage { + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub total_tokens: u64, + pub prompt_tokens_details: PromptTokensDetails, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionsChoiceMessage { + pub role: String, + // Whether an empty turn is `None` or `""` is the provider's choice, not a + // shared invariant: Anthropic's transform ends on `merged_text or None` + // while Converse assigns the joined string unconditionally. Each config + // mirrors its own, so keep this optional and serialize it even when None. + pub content: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionsChoice { + pub index: u64, + pub message: ChatCompletionsChoiceMessage, + pub finish_reason: String, +} + +/// The normalized response handed back to the host. +/// +/// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the +/// `ModelResponse` it already created, and echoing the provider's own id here +/// would change it. Pinned by `response_carries_no_id` in `tests.rs`. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionsResponse { + pub created: u64, + pub model: String, + pub choices: Vec, + pub usage: ChatCompletionsUsage, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionDelta { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thinking_blocks: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionStreamingChoice { + pub index: u64, + pub delta: ChatCompletionDelta, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub finish_reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub logprobs: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +pub struct ChatCompletionChunk { + pub id: String, + pub created: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model: Option, + pub object: String, + pub choices: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option>, +} diff --git a/litellm/__init__.py b/litellm/__init__.py index 71857877e53..09585bda7bf 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -267,10 +267,6 @@ route_all_chat_openai_to_responses: bool = ( # When True, Gemini/Vertex Live setup is deferred until client `session.update`. # Default False preserves historical behavior (auto-send setup on connect). gemini_live_defer_setup: bool = os.getenv("LITELLM_GEMINI_LIVE_DEFER_SETUP", "false").lower() == "true" -use_legacy_interactions_schema: bool = ( - os.getenv("LITELLM_USE_LEGACY_INTERACTIONS_SCHEMA", "false").lower() == "true" -) # When True, sends Api-Revision: 2026-05-07 to Google so responses use the legacy `outputs` -# schema instead of the new `steps` schema. Remove this flag after June 8, 2026. retry = True ### AUTH ### api_key: Optional[str] = None diff --git a/litellm/interactions/litellm_responses_transformation/streaming_iterator.py b/litellm/interactions/litellm_responses_transformation/streaming_iterator.py index 1a9fce5a9d7..dab12d447df 100644 --- a/litellm/interactions/litellm_responses_transformation/streaming_iterator.py +++ b/litellm/interactions/litellm_responses_transformation/streaming_iterator.py @@ -33,11 +33,8 @@ class LiteLLMResponsesInteractionsStreamingIterator: streaming events (output.text.delta, response.completed, etc.) to Interactions API streaming events. - Schema selection: - - New schema (default, use_legacy_interactions_schema=False): - interaction.created -> step.start -> step.delta ... -> step.stop -> interaction.completed - - Legacy schema (use_legacy_interactions_schema=True, remove after June 8 2026): - interaction.start -> content.start -> content.delta ... -> content.stop -> interaction.complete + Emits the event sequence + ``interaction.created -> step.start -> step.delta ... -> step.stop -> interaction.completed``. """ def __init__( @@ -49,8 +46,6 @@ class LiteLLMResponsesInteractionsStreamingIterator: custom_llm_provider: str | None = None, litellm_metadata: dict[str, Any] | None = None, ): - import litellm - self.model = model self.responses_stream_iterator = litellm_custom_stream_wrapper self.request_input = request_input @@ -61,10 +56,6 @@ class LiteLLMResponsesInteractionsStreamingIterator: self.collected_text = "" self.sent_interaction_start = False self.sent_content_start = False - # Capture the schema flag once at construction time so all events - # emitted by this stream use a consistent schema, even if the global - # flag is mutated mid-stream (e.g. by a config reload). - self._use_legacy: bool = litellm.use_legacy_interactions_schema # Buffer of events that have been derived from upstream chunks but not # yet returned to the caller. A single Responses API chunk may expand # into multiple Interactions API events (e.g. the first text delta @@ -85,9 +76,8 @@ class LiteLLMResponsesInteractionsStreamingIterator: # ------------------------------------------------------------------ def _build_interaction_start_event(self, interaction_id: str) -> InteractionsAPIStreamingResponse: - event_type: Final = "interaction.start" if self._use_legacy else "interaction.created" return InteractionsAPIStreamingResponse( - event_type=event_type, + event_type="interaction.created", id=interaction_id, object="interaction", status="in_progress", @@ -95,13 +85,6 @@ class LiteLLMResponsesInteractionsStreamingIterator: ) def _build_content_start_event(self, interaction_id: str) -> InteractionsAPIStreamingResponse: - if self._use_legacy: - return InteractionsAPIStreamingResponse( - event_type="content.start", - id=interaction_id, - object="content", - delta={"type": "text", "text": ""}, - ) return InteractionsAPIStreamingResponse( event_type="step.start", index=0, @@ -109,13 +92,6 @@ class LiteLLMResponsesInteractionsStreamingIterator: ) def _build_text_delta_event(self, interaction_id: str, delta_text: str) -> InteractionsAPIStreamingResponse: - if self._use_legacy: - return InteractionsAPIStreamingResponse( - event_type="content.delta", - id=interaction_id, - object="content", - delta={"type": "text", "text": delta_text}, - ) return InteractionsAPIStreamingResponse( event_type="step.delta", index=0, @@ -123,28 +99,12 @@ class LiteLLMResponsesInteractionsStreamingIterator: ) def _build_content_stop_event(self, interaction_id: str | None) -> InteractionsAPIStreamingResponse: - if self._use_legacy: - return InteractionsAPIStreamingResponse( - event_type="content.stop", - id=interaction_id, - object="content", - delta={"type": "text", "text": self.collected_text}, - ) return InteractionsAPIStreamingResponse( event_type="step.stop", index=0, ) def _build_completion_event(self, response_id: str) -> InteractionsAPIStreamingResponse: - if self._use_legacy: - return InteractionsAPIStreamingResponse( - event_type="interaction.complete", - id=response_id, - object="interaction", - status="completed", - model=self.model, - outputs=[{"type": "text", "text": self.collected_text}], - ) return InteractionsAPIStreamingResponse( event_type="interaction.completed", id=response_id, @@ -234,7 +194,7 @@ class LiteLLMResponsesInteractionsStreamingIterator: """ Build the events to flush when the upstream stream ends without a ResponseCompletedEvent. Ensures consumers always observe a terminal - interaction.completed/interaction.complete carrying the full text. + interaction.completed carrying the full text. """ if self._sent_completion_event: return [] diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 99b0d40f0c0..4a9a65b1485 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3914,9 +3914,8 @@ class Logging(LiteLLMLoggingBaseClass): ) -> InteractionsAPIResponse | None: """ The Interactions API streaming iterator hands the terminal event to the - success handlers: the new schema (Api-Revision: 2026-05-20) emits - ``interaction.completed`` carrying the full interaction object, the - legacy schema (2026-05-07) emits a chunk with ``status="completed"`` + success handlers: ``interaction.completed`` may carry the full + interaction object, or the final chunk may carry ``status="completed"`` and usage on the chunk itself. Build the equivalent non-streaming response so cost calculation and spend tracking see one shape. """ diff --git a/litellm/llms/gemini/interactions/transformation.py b/litellm/llms/gemini/interactions/transformation.py index 6d0f211ed7b..ab2c1440fb7 100644 --- a/litellm/llms/gemini/interactions/transformation.py +++ b/litellm/llms/gemini/interactions/transformation.py @@ -6,10 +6,7 @@ Per OpenAPI spec (https://ai.google.dev/static/api/interactions.openapi.json): - Get: GET https://generativelanguage.googleapis.com/{api_version}/interactions/{interaction_id} - Delete: DELETE https://generativelanguage.googleapis.com/{api_version}/interactions/{interaction_id} -Schema versioning: -- Default (Api-Revision: 2026-05-20): new `steps` schema. -- Legacy (Api-Revision: 2026-05-07): old `outputs` schema, controlled via - litellm.use_legacy_interactions_schema = True. Remove flag after June 8, 2026. +Requests use Api-Revision 2026-05-20 (`steps` schema). """ from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias @@ -17,7 +14,6 @@ from typing import TYPE_CHECKING, Any, Final, Protocol, TypeAlias import httpx from typing_extensions import ReadOnly, TypedDict -import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.core_helpers import process_response_headers from litellm.litellm_core_utils.url_utils import encode_url_path_segment @@ -137,13 +133,7 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): if api_key: headers["x-goog-api-key"] = api_key - # Inject the Api-Revision header to select the response schema. - # Default to the new `steps` schema unless the operator has opted out. - # Remove this conditional after June 8, 2026 and always use 2026-05-20. - if litellm.use_legacy_interactions_schema: - headers["Api-Revision"] = "2026-05-07" - else: - headers["Api-Revision"] = "2026-05-20" + headers["Api-Revision"] = "2026-05-20" return headers @@ -180,17 +170,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): """ Build request body per OpenAPI spec. - When on the new schema (use_legacy_interactions_schema=False, the default): - ``response_mime_type`` is folded into ``response_format`` and stripped from the body (the field was removed in Api-Revision 2026-05-20). - ``generation_config.image_config`` is moved to a ``response_format`` entry with ``"type": "image"`` (also removed from generation_config in 2026-05-20). - - When on the legacy schema (use_legacy_interactions_schema=True): - - All fields are forwarded as-is. """ - use_legacy: Final[bool] = litellm.use_legacy_interactions_schema - request_body: Final[dict[str, object]] = {} # Model or Agent (one required) @@ -205,7 +189,6 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): if input is not None: request_body["input"] = input - # Pass through optional params — legacy schema keeps all fields as-is. optional_keys: Final = [ "tools", "system_instruction", @@ -220,58 +203,51 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): if optional_params.get(key) is not None: request_body[key] = optional_params[key] - if use_legacy: - # Legacy schema: forward response_mime_type and response_format as-is. - for key in ("response_format", "response_mime_type", "generation_config"): - if optional_params.get(key) is not None: - request_body[key] = optional_params[key] - else: - # New schema (Api-Revision: 2026-05-20): - # response_mime_type is removed — fold it into response_format. - response_format = optional_params.get("response_format") - response_mime_type: Final = optional_params.get("response_mime_type") - - if ( - response_mime_type - and not isinstance(response_format, list) - and (not isinstance(response_format, dict) or "mime_type" not in response_format) - ): - # Wrap the legacy schema into the new polymorphic format. - new_rf: Final[dict[str, object]] = { - "type": "text", - "mime_type": response_mime_type, - } - if response_format is not None: - new_rf["schema"] = response_format - response_format = new_rf + # response_mime_type is removed — fold it into response_format. + response_format = optional_params.get("response_format") + response_mime_type: Final = optional_params.get("response_mime_type") + if ( + response_mime_type + and not isinstance(response_format, list) + and (not isinstance(response_format, dict) or "mime_type" not in response_format) + ): + # Wrap the legacy schema into the new polymorphic format. + new_rf: Final[dict[str, object]] = { + "type": "text", + "mime_type": response_mime_type, + } if response_format is not None: - request_body["response_format"] = response_format + new_rf["schema"] = response_format + response_format = new_rf + + if response_format is not None: + request_body["response_format"] = response_format + + # image_config moves out of generation_config into response_format. + generation_config: dict[str, Any] | None = optional_params.get("generation_config") + if generation_config is not None: + image_config = None + if isinstance(generation_config, dict): + generation_config = dict(generation_config) # avoid mutating the caller's dict + image_config = generation_config.pop("image_config", None) + if not generation_config: + generation_config = None - # image_config moves out of generation_config into response_format. - generation_config: dict[str, Any] | None = optional_params.get("generation_config") if generation_config is not None: - image_config = None - if isinstance(generation_config, dict): - generation_config = dict(generation_config) # avoid mutating the caller's dict - image_config = generation_config.pop("image_config", None) - if not generation_config: - generation_config = None + request_body["generation_config"] = generation_config - if generation_config is not None: - request_body["generation_config"] = generation_config - - if image_config is not None: - # Move image_config to response_format with type=image. - image_rf: Final[_JsonObject] = {"type": "image", **image_config} - existing_rf: Final = request_body.get("response_format") - if existing_rf is None: - request_body["response_format"] = image_rf - elif isinstance(existing_rf, list): - request_body["response_format"] = [*existing_rf, image_rf] - else: - # Convert single entry to array for multimodal output. - request_body["response_format"] = [existing_rf, image_rf] + if image_config is not None: + # Move image_config to response_format with type=image. + image_rf: Final[_JsonObject] = {"type": "image", **image_config} + existing_rf: Final = request_body.get("response_format") + if existing_rf is None: + request_body["response_format"] = image_rf + elif isinstance(existing_rf, list): + request_body["response_format"] = [*existing_rf, image_rf] + else: + # Convert single entry to array for multimodal output. + request_body["response_format"] = [existing_rf, image_rf] return request_body diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 7191a33a74a..dc1121977ec 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -41100,21 +41100,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.4336e-07, + "input_cost_per_token": 1.6e-06, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 1.88672e-06, + "output_cost_per_token": 3.2e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.9596e-08, + "cache_read_input_token_cost": 1.35e-07, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -41141,21 +41141,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro-0813": { - "input_cost_per_token": 6.6e-07, + "input_cost_per_token": 5.7816e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 1.98e-06, + "output_cost_per_token": 1.73448e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost": 1.8396e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -41197,6 +41197,7 @@ "supports_vision": true, "supports_image_size": false, "cache_read_input_token_cost": 3e-08, + "deprecation_date": "2026-10-20", "supports_prompt_caching": true, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": true, @@ -41223,6 +41224,7 @@ "supports_vision": true, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "deprecation_date": "2026-10-20", "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 1.5e-05, "supports_prompt_caching": true, @@ -65084,6 +65086,7 @@ "supports_pdf_input": true, "supports_audio_input": true, "cache_read_input_token_cost": 1e-08, + "deprecation_date": "2026-10-20", "input_cost_per_audio_token": 3e-07, "supports_prompt_caching": true, "supports_web_search": false @@ -66130,9 +66133,9 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.2": { - "input_cost_per_token": 4.875e-07, - "output_cost_per_token": 1.56e-06, - "cache_read_input_token_cost": 9.1e-08, + "input_cost_per_token": 5.544e-07, + "output_cost_per_token": 1.7424e-06, + "cache_read_input_token_cost": 1.0296e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 131072, @@ -66492,9 +66495,9 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 8.8606e-08, - "output_cost_per_token": 1.77212e-07, - "cache_read_input_token_cost": 1.77212e-08, + "input_cost_per_token": 4.984e-08, + "output_cost_per_token": 9.968e-08, + "cache_read_input_token_cost": 9.968e-09, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -67162,6 +67165,7 @@ "cache_read_input_token_cost": 3e-08, "cache_creation_input_token_cost": 8.33333333333333e-08, "cache_read_input_audio_token_cost": 1e-07, + "deprecation_date": "2027-03-15", "input_cost_per_audio_token": 1e-06, "output_cost_per_image_token": 3e-05, "litellm_provider": "openrouter", @@ -67401,7 +67405,7 @@ "supports_audio_input": false, "supports_function_calling": true, "supports_pdf_input": false, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_reasoning": false, "supports_tool_choice": true, "supports_response_schema": true, @@ -70545,14 +70549,14 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-pro-latest": { - "cache_read_input_token_cost": 2.2e-08, - "input_cost_per_token": 6.6e-07, + "cache_read_input_token_cost": 1.8396e-08, + "input_cost_per_token": 5.7816e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 1.98e-06, + "output_cost_per_token": 1.73448e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -70565,14 +70569,14 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-v4-flash-latest": { - "cache_read_input_token_cost": 8.8e-09, - "input_cost_per_token": 5.58e-08, + "cache_read_input_token_cost": 1.75e-09, + "input_cost_per_token": 5.5e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 1.767e-07, + "output_cost_per_token": 1.65e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -70817,14 +70821,14 @@ "supports_web_search": false }, "openrouter/~z-ai/glm-latest": { - "cache_read_input_token_cost": 1.755e-07, - "input_cost_per_token": 8.775e-07, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 9e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 235929, "max_tokens": 235929, "mode": "chat", - "output_cost_per_token": 2.97e-06, + "output_cost_per_token": 3e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -71699,6 +71703,7 @@ "openrouter/google/gemini-2.5-flash:batch": { "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 3e-08, + "deprecation_date": "2026-10-20", "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 1.5e-07, "litellm_provider": "openrouter", @@ -71722,6 +71727,7 @@ "cache_read_input_audio_token_cost": 1.25e-07, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "deprecation_date": "2026-10-20", "input_cost_per_audio_token": 6.25e-07, "input_cost_per_token": 6.25e-07, "input_cost_per_token_above_200k_tokens": 1.25e-06, @@ -72237,13 +72243,13 @@ }, "openrouter/meta/muse-glimmer-30b": { "cache_read_input_token_cost": 4e-08, - "input_cost_per_token": 3e-07, + "input_cost_per_token": 3.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 131072, "max_output_tokens": 117964, "max_tokens": 117964, "mode": "chat", - "output_cost_per_token": 1.1e-06, + "output_cost_per_token": 1.5e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index ffb27d5f92e..7733ad1c522 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -2579,7 +2579,7 @@ def _jwt_auth_issuers() -> list: if env_issuer: issuers.append(env_issuer) - jwtauth: Final = general_settings.get("litellm_jwtauth") if isinstance(general_settings, dict) else None + jwtauth: Final = general_settings.get("litellm_jwtauth") if isinstance(general_settings, Mapping) else None raw_issuers: Final = jwtauth.get("issuers") if isinstance(jwtauth, dict) else getattr(jwtauth, "issuers", None) for cfg in raw_issuers or []: issuer = cfg.get("issuer") if isinstance(cfg, dict) else getattr(cfg, "issuer", None) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index b0d31df92ce..aacf318267a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2423,6 +2423,8 @@ class ConfigList(LiteLLMPydanticObjectBase): nested_fields: list[FieldDetail] | None = None # For nested dictionary or Pydantic fields field_options: list[str] | None = None # Allowed values, for field_type == "Select" field_tab: str | None = None # Admin UI sub-tab this field renders under; None groups it with the rest + source: Literal["config", "db", "env", "default", "unset"] = "unset" + editable: bool = True class UserHeaderMapping(LiteLLMPydanticObjectBase): @@ -3693,6 +3695,8 @@ class InvitationClaim(LiteLLMPydanticObjectBase): class ConfigFieldInfo(LiteLLMPydanticObjectBase): field_name: str field_value: Any + source: Literal["config", "db", "env", "default", "unset"] = "unset" + editable: bool = True class CallbackOnUI(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 3dd2e2d8eb2..ca023a06f39 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -3637,6 +3637,22 @@ async def get_jwt_key_mapping_cache_keys_for_token( return tuple(jwt_key_mapping_cache_key(m.jwt_claim_name, m.jwt_claim_value, m.jwt_issuer) for m in mappings) +class _TokenInFilter(TypedDict): + token: ReadOnly[Mapping[str, Sequence[str]]] + + +async def get_jwt_key_mapping_cache_keys_for_tokens( + hashed_tokens: Sequence[str], + prisma_client: PrismaClient, +) -> tuple[str, ...]: + """Cache keys of every JWT claim mapped to any of the given virtual keys.""" + if not hashed_tokens: + return () + token_filter: Final[_TokenInFilter] = {"token": {"in": tuple(hashed_tokens)}} + mappings: Final = await _jwt_key_mapping_table(JWTKeyMappingRepository(prisma_client)).find_many(where=token_filter) + return tuple(jwt_key_mapping_cache_key(m.jwt_claim_name, m.jwt_claim_value, m.jwt_issuer) for m in mappings) + + @log_db_metrics async def get_jwt_key_mapping_object( jwt_claim_name: str, diff --git a/litellm/proxy/client/cli/commands/autoroute/commands.py b/litellm/proxy/client/cli/commands/autoroute/commands.py index 05c21875f84..d9fc3ae9f77 100644 --- a/litellm/proxy/client/cli/commands/autoroute/commands.py +++ b/litellm/proxy/client/cli/commands/autoroute/commands.py @@ -2,6 +2,7 @@ import atexit import secrets import signal import threading +from collections.abc import Mapping from types import FrameType from typing import Final @@ -67,7 +68,7 @@ def _ensure_master_key() -> str: master_key: Final = secrets.token_urlsafe(32) general_settings: Final = generated.get("general_settings") updated_settings: Final[dict[str, JsonValue]] = { - **(general_settings if isinstance(general_settings, dict) else {}), + **(general_settings if isinstance(general_settings, Mapping) else {}), "master_key": master_key, } updated: Final[dict[str, JsonValue]] = {**generated, "general_settings": updated_settings} diff --git a/litellm/proxy/client/cli/commands/autoroute/config.py b/litellm/proxy/client/cli/commands/autoroute/config.py index 1bfcdf444bf..f8738af221e 100644 --- a/litellm/proxy/client/cli/commands/autoroute/config.py +++ b/litellm/proxy/client/cli/commands/autoroute/config.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from typing import Final, Literal from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter @@ -221,7 +222,7 @@ def master_key_from_config(config: dict[str, JsonValue]) -> str | None: normalized copy here would diverge from what the proxy expects. """ general_settings: Final = config.get("general_settings") - if not isinstance(general_settings, dict): + if not isinstance(general_settings, Mapping): return None master_key: Final = general_settings.get("master_key") if isinstance(master_key, str) and master_key.strip(): diff --git a/litellm/proxy/config_resolvers/__init__.py b/litellm/proxy/config_resolvers/__init__.py index 88b4c3961f0..ebd339b34c3 100644 --- a/litellm/proxy/config_resolvers/__init__.py +++ b/litellm/proxy/config_resolvers/__init__.py @@ -5,5 +5,6 @@ from litellm.proxy.config_resolvers._descriptors import ( FieldSource, resolve_fields, ) +from litellm.proxy.config_resolvers.settings_store import SettingsStore -__all__ = ["FieldDescriptor", "FieldSource", "resolve_fields"] +__all__ = ("FieldDescriptor", "FieldSource", "SettingsStore", "resolve_fields") diff --git a/litellm/proxy/config_resolvers/_descriptors.py b/litellm/proxy/config_resolvers/_descriptors.py index edc0eeb1cf6..e2e0534bf7b 100644 --- a/litellm/proxy/config_resolvers/_descriptors.py +++ b/litellm/proxy/config_resolvers/_descriptors.py @@ -13,7 +13,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Final, Literal -FieldSource = Literal["db", "env", "default", "unset"] +FieldSource = Literal["config", "db", "env", "default", "unset"] @dataclass(frozen=True, slots=True) @@ -69,5 +69,7 @@ def resolve_fields( """ resolved: Final = tuple(_resolve_one(descriptor, db_values, env, empty_db_is_set) for descriptor in descriptors) values: Final = {field_name: value for field_name, value, _ in resolved} - provenance: Final = {field_name: source for field_name, _, source in resolved} + provenance: Final[dict[str, FieldSource]] = dict( # mutable-ok: public resolver contract returns a plain dict + (field_name, source) for field_name, _, source in resolved + ) return values, provenance diff --git a/litellm/proxy/config_resolvers/settings_rules.py b/litellm/proxy/config_resolvers/settings_rules.py new file mode 100644 index 00000000000..f346dd6198d --- /dev/null +++ b/litellm/proxy/config_resolvers/settings_rules.py @@ -0,0 +1,110 @@ +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +from litellm.proxy.config_resolvers._descriptors import FieldSource + +JsonValue: TypeAlias = None | bool | int | float | str | list["JsonValue"] | dict[str, "JsonValue"] +Section: TypeAlias = Literal[ + "general_settings", + "router_settings", + "litellm_settings", + "environment_variables", + "ui_settings", +] +DbRow: TypeAlias = Section + + +@dataclass(frozen=True, slots=True) +class Absent: + pass + + +ABSENT: Final = Absent() +SettingValue: TypeAlias = JsonValue | Absent + + +@dataclass(frozen=True, slots=True) +class KeyRule: + """Which stored row carries this key. Precedence no longer varies per key.""" + + db_row: DbRow + + +@dataclass(frozen=True, slots=True) +class Resolved: + value: SettingValue + source: FieldSource + + +_UI_SETTINGS_FIELDS: Final[tuple[str, ...]] = ( + "allow_public_health_readiness_details", + "forward_client_headers_to_llm_api", + "forward_llm_provider_auth_headers", + "disable_agents_for_internal_users", + "allow_agents_for_team_admins", + "disable_vector_stores_for_internal_users", + "allow_vector_stores_for_team_admins", + "disable_key_generate_for_org_admin", + "team_admin_editable_team_fields", +) + + +def _rules_for( + section: Section, keys: tuple[str, ...], db_row: DbRow +) -> tuple[tuple[tuple[Section, str], KeyRule], ...]: + return tuple(((section, key), KeyRule(db_row=db_row)) for key in keys) + + +def _build_dual_source_keys() -> Mapping[tuple[Section, str], KeyRule]: + """Maps a key to the stored row that carries it, for the keys whose row is not their own section.""" + return MappingProxyType( + dict( + ( + *_rules_for("general_settings", _UI_SETTINGS_FIELDS, "ui_settings"), + *( + ((section, "*"), KeyRule(db_row=section)) + for section in ("general_settings", "router_settings", "litellm_settings", "environment_variables") + ), + ) + ) + ) + + +DUAL_SOURCE_KEYS: Final[Mapping[tuple[Section, str], KeyRule]] = _build_dual_source_keys() + + +def rule_for(section: Section, key: str) -> KeyRule: + return DUAL_SOURCE_KEYS.get((section, key), DUAL_SOURCE_KEYS[(section, "*")]) + + +def coerce_bool(value: JsonValue) -> JsonValue: + if value is None or isinstance(value, bool): + return value + if isinstance(value, str): + return value.lower() == "true" + return bool(value) + + +def resolve(yaml_value: SettingValue, db_value: SettingValue) -> Resolved: + """Config wins. A key the config file declares is config-owned, whatever the database holds. + + A stored ``null`` still counts as absent, so clearing a row does not erase a value + the file never declared. + """ + if yaml_value is not ABSENT: + return Resolved(value=yaml_value, source="config") + if _db_is_present(db_value): + return Resolved(value=db_value, source="db") + return Resolved(value=ABSENT, source="unset") + + +def is_absent(value: SettingValue) -> bool: + return value is ABSENT + + +def _db_is_present(value: SettingValue) -> bool: + return not is_absent(value) and value is not None diff --git a/litellm/proxy/config_resolvers/settings_store.py b/litellm/proxy/config_resolvers/settings_store.py new file mode 100644 index 00000000000..079d262319f --- /dev/null +++ b/litellm/proxy/config_resolvers/settings_store.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from collections.abc import Iterator, Mapping, MutableMapping +from types import MappingProxyType +from typing import Final + +from litellm.proxy.config_resolvers._descriptors import FieldSource +from litellm.proxy.config_resolvers.settings_rules import ( + ABSENT, + Absent, + DbRow, + JsonValue, + Resolved, + Section, + SettingValue, + resolve, + rule_for, +) + +_EMPTY_VALUES: Final[Mapping[str, JsonValue]] = MappingProxyType({}) +_EMPTY_ROWS: Final[Mapping[DbRow, Mapping[str, JsonValue]]] = MappingProxyType({}) + + +class SettingsStore(MutableMapping[str, JsonValue]): + def __init__(self, section: Section) -> None: + self._section: Final = section + self._yaml_values: Mapping[str, JsonValue] = _EMPTY_VALUES + self._database_rows: Mapping[DbRow, Mapping[str, JsonValue]] = _EMPTY_ROWS + self._runtime_values: Mapping[str, JsonValue] = _EMPTY_VALUES + self._deleted_runtime_keys: frozenset[str] = frozenset() + + def load_yaml(self, mapping: Mapping[str, JsonValue]) -> None: + self._yaml_values = MappingProxyType(dict(mapping)) + self._clear_runtime() + + def config_value(self, key: str) -> JsonValue: + return self._yaml_values.get(key) + + def owned_by_config(self, key: str) -> bool: + return key in self._yaml_values + + def rejected_writes(self, incoming: Mapping[str, JsonValue]) -> tuple[str, ...]: + return tuple( + sorted( + key for key, value in incoming.items() if self.owned_by_config(key) and value != self._yaml_values[key] + ) + ) + + def apply_db_row(self, row: DbRow, db_row: Mapping[str, JsonValue]) -> None: + previous_row: Final = self._database_rows.get(row, _EMPTY_VALUES) + self._database_rows = MappingProxyType({**self._database_rows, row: MappingProxyType(dict(db_row))}) + self._clear_runtime_keys(frozenset((*previous_row, *db_row))) + + def resolved(self) -> Mapping[str, JsonValue]: + return MappingProxyType(dict(self)) + + def apply_runtime_values(self, values: Mapping[str, JsonValue]) -> None: + self._runtime_values = MappingProxyType(dict(values)) + self._deleted_runtime_keys = frozenset() + + def source(self, key: str) -> FieldSource: + return self._resolution_for(key).source + + def __getitem__(self, key: str) -> JsonValue: + if key in self._deleted_runtime_keys: + raise KeyError(key) + if key in self._runtime_values: + return self._runtime_values[key] + resolved: Final = self._resolution_for(key) + if isinstance(resolved.value, Absent): + raise KeyError(key) + return resolved.value + + def __setitem__(self, key: str, value: JsonValue) -> None: + if self.owned_by_config(key): + return + self._runtime_values = MappingProxyType({**self._runtime_values, key: value}) + self._deleted_runtime_keys = self._deleted_runtime_keys - frozenset((key,)) + + def __delitem__(self, key: str) -> None: + if key not in self: + raise KeyError(key) + if self.owned_by_config(key): + return + self._runtime_values = MappingProxyType( + {key_: value for key_, value in self._runtime_values.items() if key_ != key} + ) + self._deleted_runtime_keys = self._deleted_runtime_keys | frozenset((key,)) + + def __iter__(self) -> Iterator[str]: + return iter( + key + for key in self._keys() + if key not in self._deleted_runtime_keys + and (key in self._runtime_values or not isinstance(self._resolution_for(key).value, Absent)) + ) + + def __len__(self) -> int: + return sum(1 for _ in self) + + def _clear_runtime(self) -> None: + self._runtime_values = _EMPTY_VALUES + self._deleted_runtime_keys = frozenset() + + def _clear_runtime_keys(self, keys: frozenset[str]) -> None: + if not keys: + return + self._runtime_values = MappingProxyType( + {key: value for key, value in self._runtime_values.items() if key not in keys} + ) + self._deleted_runtime_keys = self._deleted_runtime_keys - keys + + def _keys(self) -> tuple[str, ...]: + return tuple( + dict.fromkeys( + ( + *self._yaml_values, + *(key for row in self._database_rows.values() for key in row), + *self._runtime_values, + ) + ) + ) + + def _resolution_for(self, key: str) -> Resolved: + rule: Final = rule_for(self._section, key) + yaml_value: Final[SettingValue] = self._yaml_values.get(key, ABSENT) + db_value: Final[SettingValue] = self._database_rows.get(rule.db_row, _EMPTY_VALUES).get(key, ABSENT) + return resolve(yaml_value, db_value) diff --git a/litellm/proxy/management_endpoints/coordination_redis_endpoints.py b/litellm/proxy/management_endpoints/coordination_redis_endpoints.py index 88dc09ab001..8e64e1ea651 100644 --- a/litellm/proxy/management_endpoints/coordination_redis_endpoints.py +++ b/litellm/proxy/management_endpoints/coordination_redis_endpoints.py @@ -198,7 +198,7 @@ async def _current_coordination_redis_settings() -> dict[str, object] | None: config_state: Final = _SETTINGS_ADAPTER.validate_python(proxy_config.get_config_state()) general_settings: Final = config_state.get(_GENERAL_SETTINGS_PARAM_NAME) - if not isinstance(general_settings, dict): + if not isinstance(general_settings, Mapping): return None from_file: Final = general_settings.get(_COORDINATION_REDIS_KEY) if isinstance(from_file, dict): diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index ba7a3309a90..4832c2f4c21 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -23,12 +23,18 @@ from typing import Any, Final, Literal, Protocol, cast, overload import fastapi from fastapi import APIRouter, Depends, Header, HTTPException, Request, status from pydantic import TypeAdapter, ValidationError +from typing_extensions import ReadOnly, TypedDict import litellm from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.proxy._types import * -from litellm.proxy.auth.auth_checks import get_team_object, get_user_object +from litellm.proxy.auth.auth_checks import ( + delete_cache_key_objects, + get_jwt_key_mapping_cache_keys_for_tokens, + get_team_object, + get_user_object, +) from litellm.proxy.auth.password_policy import validate_password_policy from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast @@ -126,6 +132,10 @@ def _verification_token_table( return token_table +class _UserIdInFilter(TypedDict): + user_id: ReadOnly[Mapping[str, Sequence[str]]] + + def _organization_membership_table( prisma_client: "PrismaClient | None", ) -> "TableActions[prisma_models.LiteLLM_OrganizationMembership]": @@ -2345,6 +2355,8 @@ async def delete_user( create_audit_log_for_update, litellm_proxy_admin_name, prisma_client, + proxy_logging_obj, + user_api_key_cache, ) if prisma_client is None: @@ -2471,7 +2483,20 @@ async def delete_user( # End of Audit logging ## DELETE ASSOCIATED KEYS - await _verification_token_table(prisma_client).delete_many(where={"user_id": {"in": data.user_ids}}) + key_filter: Final[_UserIdInFilter] = {"user_id": {"in": data.user_ids}} + keys_to_delete: Final = await _verification_token_table(prisma_client).find_many(where=key_filter) + hashed_tokens_to_delete: Final = tuple(key.token for key in keys_to_delete) + jwt_mapping_cache_keys: Final = await get_jwt_key_mapping_cache_keys_for_tokens( + hashed_tokens=hashed_tokens_to_delete, + prisma_client=prisma_client, + ) + await _verification_token_table(prisma_client).delete_many(where=key_filter) + await delete_cache_key_objects( + hashed_tokens=hashed_tokens_to_delete, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + await evict_and_broadcast(cache_keys=jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache) ## DELETE ASSOCIATED INVITATION LINKS await _invitation_link_table(prisma_client).delete_many( diff --git a/litellm/proxy/management_endpoints/organization_endpoints.py b/litellm/proxy/management_endpoints/organization_endpoints.py index c6a76a920f6..29c31cc9c18 100644 --- a/litellm/proxy/management_endpoints/organization_endpoints.py +++ b/litellm/proxy/management_endpoints/organization_endpoints.py @@ -26,13 +26,21 @@ from typing import ( import fastapi from fastapi import APIRouter, Depends, HTTPException, Request, status from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid from litellm.proxy._types import * -from litellm.proxy.auth.auth_checks import can_user_call_model, get_user_object +from litellm.proxy.auth.auth_checks import ( + can_user_call_model, + delete_cache_key_objects, + get_jwt_key_mapping_cache_keys_for_tokens, + get_user_object, +) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.management_endpoints.budget_management_endpoints import ( new_budget, update_budget, @@ -52,7 +60,7 @@ from litellm.proxy.management_helpers.utils import ( get_new_internal_user_defaults, management_endpoint_wrapper, ) -from litellm.proxy.utils import PrismaClient +from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.repositories.budget_repository import BudgetRepository from litellm.repositories.object_permission_repository import ObjectPermissionRepository from litellm.repositories.organization_repository import OrganizationRepository @@ -79,6 +87,7 @@ if TYPE_CHECKING: ) from prisma.models import LiteLLM_OrganizationTable as PrismaOrganizationTable from prisma.models import LiteLLM_UserTable as PrismaUserTable + from prisma.models import LiteLLM_VerificationToken as PrismaVerificationToken async def _enterprise_license_required( @@ -168,9 +177,15 @@ class _TeamTableClient(Protocol): class _VerificationTokenTableClient(Protocol): + async def find_many(self, where: Mapping[str, object] | None = None) -> "Sequence[PrismaVerificationToken]": ... + async def delete_many(self, where: Mapping[str, object]) -> int: ... +class _OrganizationIdFilter(TypedDict): + organization_id: ReadOnly[str] + + class _ObjectPermissionTxClient(Protocol): async def upsert( self, where: Mapping[str, object], data: Mapping[str, object] @@ -961,7 +976,7 @@ async def delete_organization( - organization_ids: List[str] - The organization ids to delete. """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache if prisma_client is None: raise HTTPException( @@ -983,8 +998,12 @@ async def delete_organization( await _table(OrganizationMembershipRepository(prisma_client)).delete_many( where={"organization_id": organization_id} ) - # delete all keys in the organization - await _table(VerificationTokenRepository(prisma_client)).delete_many(where={"organization_id": organization_id}) + await _delete_organization_keys( + organization_id=organization_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) # delete the organization deleted_org = await _table(OrganizationRepository(prisma_client)).delete( where={"organization_id": organization_id}, @@ -1000,6 +1019,28 @@ async def delete_organization( return deleted_orgs +async def _delete_organization_keys( + organization_id: str, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, + proxy_logging_obj: ProxyLogging | None, +) -> None: + key_filter: Final[_OrganizationIdFilter] = {"organization_id": organization_id} + keys_to_delete: Final = await _table(VerificationTokenRepository(prisma_client)).find_many(where=key_filter) + hashed_tokens_to_delete: Final = tuple(key.token for key in keys_to_delete) + jwt_mapping_cache_keys: Final = await get_jwt_key_mapping_cache_keys_for_tokens( + hashed_tokens=hashed_tokens_to_delete, + prisma_client=prisma_client, + ) + await _table(VerificationTokenRepository(prisma_client)).delete_many(where=key_filter) + await delete_cache_key_objects( + hashed_tokens=hashed_tokens_to_delete, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + await evict_and_broadcast(cache_keys=jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache) + + @router.get( "/organization/list", tags=["organization management"], diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 216480e298b..b37d7970e50 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -99,6 +99,7 @@ from litellm.proxy.auth.auth_checks import ( can_org_access_model, delete_cache_key_objects, delete_cache_team_object, + get_jwt_key_mapping_cache_keys_for_tokens, get_org_object, get_team_membership, get_team_object, @@ -110,6 +111,7 @@ from litellm.proxy.auth.auth_utils import ( enforce_output_token_estimates_are_admin_only, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars from litellm.proxy.common_utils.json_merge_patch import apply_json_merge_patch from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache @@ -3524,7 +3526,6 @@ async def team_member_delete( }' ``` """ - from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast from litellm.proxy.proxy_server import prisma_client, proxy_logging_obj, user_api_key_cache if prisma_client is None: @@ -3626,6 +3627,10 @@ async def team_member_delete( "team_id": data.team_id, } ) + jwt_mapping_cache_keys: Final = await get_jwt_key_mapping_cache_keys_for_tokens( + hashed_tokens=tuple(key.token for key in keys_to_delete), + prisma_client=prisma_client, + ) if removed_team_members: await _team_tx_db(tx).update( @@ -3674,6 +3679,7 @@ async def team_member_delete( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + await evict_and_broadcast(cache_keys=jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache) await evict_and_broadcast(cache_keys=tuple(sorted(user_ids_to_delete)), user_api_key_cache=user_api_key_cache) for user_id in sorted(user_ids_to_delete): await invalidate_team_member_spend_state( @@ -4264,6 +4270,10 @@ async def delete_team( ) keys_to_delete: Final = await _tokens_db(prisma_client).find_many(where={"team_id": {"in": data.team_ids}}) + jwt_mapping_cache_keys: Final = await get_jwt_key_mapping_cache_keys_for_tokens( + hashed_tokens=tuple(key.token for key in keys_to_delete), + prisma_client=prisma_client, + ) if keys_to_delete: await _persist_deleted_verification_tokens( @@ -4280,6 +4290,7 @@ async def delete_team( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + await evict_and_broadcast(cache_keys=jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache) ## DELETE ASSOCIATED BYOK MODELS # Runs before the team rows are deleted so a mid-flight failure never leaves diff --git a/litellm/proxy/management_helpers/bulk_user_deletion.py b/litellm/proxy/management_helpers/bulk_user_deletion.py index 1ae83b0004a..af51a194413 100644 --- a/litellm/proxy/management_helpers/bulk_user_deletion.py +++ b/litellm/proxy/management_helpers/bulk_user_deletion.py @@ -28,7 +28,7 @@ from litellm.proxy._types import ( MemberDeleteRequest, UserAPIKeyAuth, ) -from litellm.proxy.auth.auth_checks import delete_cache_key_objects +from litellm.proxy.auth.auth_checks import delete_cache_key_objects, get_jwt_key_mapping_cache_keys_for_tokens from litellm.proxy.common_utils.auth_cache_invalidation_pubsub import evict_and_broadcast from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.hooks.user_management_event_hooks import UserManagementEventHooks @@ -94,12 +94,20 @@ class _TeamRemoval: removed: frozenset[str] matched: frozenset[int] deleted_key_tokens: tuple[str, ...] + jwt_mapping_cache_keys: tuple[str, ...] @dataclass(frozen=True, slots=True) class _UserBatchDeletion: removals: Mapping[str, _TeamRemoval] deleted_key_tokens: tuple[str, ...] + jwt_mapping_cache_keys: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class _DeletedKeys: + tokens: tuple[str, ...] + jwt_mapping_cache_keys: tuple[str, ...] def _team_not_found(team_id: str) -> ManagementProblem: @@ -237,6 +245,10 @@ async def _remove_members_from_team( if any(_addresses_member(m, r) for m in removed_members) or any(_addresses_user(u, r) for u in stale_rows) ) keys: Final = await _token_tx_db(tx).find_many(where=_team_users_filter(team_id, cleanup_ids)) + jwt_mapping_cache_keys: Final = await get_jwt_key_mapping_cache_keys_for_tokens( + hashed_tokens=tuple(k.token for k in keys), + prisma_client=prisma_client, + ) if removed_members: roster_data: Final[_RosterData] = { @@ -265,6 +277,7 @@ async def _remove_members_from_team( removed=cleanup_ids, matched=matched, deleted_key_tokens=tuple(k.token for k in keys), + jwt_mapping_cache_keys=jwt_mapping_cache_keys, ) @@ -322,6 +335,7 @@ async def bulk_remove_team_members( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + await evict_and_broadcast(cache_keys=removal.jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache) _emit_team_members_metric(removal.team) matched: Final = frozenset(kept_indexes[j] for j in removal.matched) @@ -368,8 +382,12 @@ async def _delete_user_rows( user_ids: frozenset[str], user_api_key_dict: UserAPIKeyAuth, litellm_changed_by: str | None, -) -> tuple[str, ...]: +) -> _DeletedKeys: keys: Final = await _token_tx_db(tx).find_many(where=_in_filter("user_id", user_ids)) + jwt_mapping_cache_keys: Final = await get_jwt_key_mapping_cache_keys_for_tokens( + hashed_tokens=tuple(k.token for k in keys), + prisma_client=prisma_client, + ) if keys: await _persist_deleted_verification_tokens( keys=keys, # pyright: ignore[reportArgumentType] # generated row model carries the same columns as LiteLLM_VerificationToken @@ -389,7 +407,7 @@ async def _delete_user_rows( await _org_membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) await _membership_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) await _user_tx_db(tx).delete_many(where=_in_filter("user_id", user_ids)) - return tuple(k.token for k in keys) + return _DeletedKeys(tokens=tuple(k.token for k in keys), jwt_mapping_cache_keys=jwt_mapping_cache_keys) async def _delete_users_tx( @@ -423,12 +441,14 @@ async def _delete_users_tx( for tid in team_ids } ) - deleted_key_tokens: Final = await _delete_user_rows( + deleted_keys: Final = await _delete_user_rows( prisma_client, tx, frozenset(u.user_id for u in users), user_api_key_dict, litellm_changed_by ) return _UserBatchDeletion( removals=removals, - deleted_key_tokens=deleted_key_tokens + tuple(t for r in removals.values() for t in r.deleted_key_tokens), + deleted_key_tokens=deleted_keys.tokens + tuple(t for r in removals.values() for t in r.deleted_key_tokens), + jwt_mapping_cache_keys=deleted_keys.jwt_mapping_cache_keys + + tuple(k for r in removals.values() for k in r.jwt_mapping_cache_keys), ) @@ -454,6 +474,7 @@ async def _delete_users( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + await evict_and_broadcast(cache_keys=deletion.jwt_mapping_cache_keys, user_api_key_cache=user_api_key_cache) await evict_and_broadcast(cache_keys=sorted(user_ids), user_api_key_cache=user_api_key_cache) for removal in deletion.removals.values(): _emit_team_members_metric(removal.team) @@ -534,7 +555,7 @@ async def bulk_delete_users( litellm_changed_by, ) if candidates - else _UserBatchDeletion(removals=MappingProxyType({}), deleted_key_tokens=()) + else _UserBatchDeletion(removals=MappingProxyType({}), deleted_key_tokens=(), jwt_mapping_cache_keys=()) ) def result(index: int, user_id: str) -> UserDeleteResult: diff --git a/litellm/proxy/plugin_routes.py b/litellm/proxy/plugin_routes.py index a72ecd4b0f2..eb6fe7dd177 100644 --- a/litellm/proxy/plugin_routes.py +++ b/litellm/proxy/plugin_routes.py @@ -67,7 +67,7 @@ def _configured_key_header_names() -> frozenset[str]: except Exception: return frozenset() general_settings: Final = getattr(proxy_server, "general_settings", None) - if not isinstance(general_settings, dict): + if not isinstance(general_settings, Mapping): return frozenset() name: Final[object] = general_settings.get("litellm_key_header_name") return frozenset({name.lower()}) if isinstance(name, str) and name else frozenset() diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 3c7a2d57fa8..6f191a42dcc 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -432,20 +432,25 @@ from litellm.proxy.common_utils.user_api_key_cache import ( model_access_group_spend_counter_key, tag_cache_key, ) -from litellm.proxy.config_resolvers import resolve_fields +from litellm.proxy.config_resolvers import SettingsStore, resolve_fields from litellm.proxy.config_resolvers.alerting import ( EMAIL_DESCRIPTORS, MS_TEAMS_DESCRIPTORS, SLACK_DESCRIPTORS, ) from litellm.proxy.config_resolvers.changed_section_keys import changed_section_keys +from litellm.proxy.config_resolvers.settings_rules import ( + DbRow, + Section, + coerce_bool, +) +from litellm.proxy.config_resolvers.settings_rules import ( + JsonValue as SettingsJsonValue, +) from litellm.proxy.container_endpoints.endpoints import router as container_router from litellm.proxy.credential_endpoints.endpoints import router as credential_router from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager -from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import ( - SPEND_LOG_CLEANUP_BOUND_SETTINGS, - SpendLogCleanup, -) +from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import SpendLogCleanup from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( build_window_spend_transaction, ) @@ -4324,7 +4329,7 @@ def _scrub_guardrail_inner(inner: dict[str, JsonValue]) -> None: inner["guardrail"] = None -def _scrub_db_overlay_remote_module_loads(section: str, db_value: JsonValue) -> JsonValue: +def _scrub_db_overlay_remote_module_loads(section: str, db_value: object) -> object: """Strip ``s3://`` / ``gcs://`` entries from the DB-overlay value for fields whose contents reach ``get_instance_fn``. The same scheme is allowed from a YAML config (the documented operator flow) but a @@ -4804,6 +4809,21 @@ class _ConfigWithBaseline(dict[str, object]): self._baseline = MappingProxyType({key: copy.deepcopy(value) for key, value in config.items()}) +_EMPTY_SETTINGS_MAPPING: Final[Mapping[str, SettingsJsonValue]] = MappingProxyType({}) +_SETTINGS_MAPPING: Final = TypeAdapter(dict[str, SettingsJsonValue]) + + +def _as_settings_mapping(value: object) -> Mapping[str, SettingsJsonValue]: + if not isinstance(value, Mapping): + return _EMPTY_SETTINGS_MAPPING + return _SETTINGS_MAPPING.validate_python(value) + + +def _bind_general_settings_store(settings: SettingsStore) -> None: + global general_settings + general_settings = settings # pyright: ignore[reportAssignmentType] # legacy global accepts mappings + + class ProxyConfig: """ Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic. @@ -4827,11 +4847,45 @@ class ProxyConfig: # whether an existing request predates the prices it just fetched, and re-serving one # costs a single fetch where skipping one leaves it priced wrong indefinitely self.model_cost_map_applied_revision: int = 0 - # Keys explicitly set in the YAML config file. Used to give YAML - # precedence over stale DB-cached values for these specific keys - # during periodic config reloads (_update_general_settings). - self._yaml_general_settings_keys: set[str] = set() # mutable-ok: populated once at startup, read-only thereafter # fmt: skip - self._yaml_spend_log_cleanup_bounds: dict[str, object] = {} # mutable-ok: snapshot of YAML bounds at load time # fmt: skip + self.settings: Final[SettingsStore] = SettingsStore("general_settings") + self.router_settings: Final[SettingsStore] = SettingsStore("router_settings") + self.litellm_settings: Final[SettingsStore] = SettingsStore("litellm_settings") + self.environment_variables: Final[SettingsStore] = SettingsStore("environment_variables") + self._settings_stores: Final[Mapping[Section, SettingsStore]] = MappingProxyType( + { + "general_settings": self.settings, + "router_settings": self.router_settings, + "litellm_settings": self.litellm_settings, + "environment_variables": self.environment_variables, + } + ) + + def _load_yaml_settings_stores(self, config: Mapping[str, object]) -> None: + global config_passthrough_endpoints + for section, store in self._settings_stores.items(): + store.load_yaml(_as_settings_mapping(config.get(section))) + store.apply_db_row(section, _EMPTY_SETTINGS_MAPPING) + yaml_endpoints: Final = self.settings.config_value("pass_through_endpoints") + config_passthrough_endpoints = ( + [dict(endpoint) for endpoint in yaml_endpoints if isinstance(endpoint, dict)] + if isinstance(yaml_endpoints, list) + else None + ) + + def _config_with_resolved_settings(self, config: Mapping[str, object]) -> dict[str, object]: + return { # mutable-ok: get_config preserves the mutable mapping contract used by existing loaders + **config, + **{ + section: dict(store.resolved()) + for section, store in self._settings_stores.items() + if isinstance(config.get(section), Mapping) or len(store) > 0 + }, + } + + def _apply_resolved_runtime_settings(self, config: Mapping[str, object]) -> None: + for section, store in self._settings_stores.items(): + if isinstance(config.get(section), Mapping): + store.apply_runtime_values(_as_settings_mapping(config[section])) def is_yaml(self, config_file_path: str) -> bool: if not os.path.isfile(config_file_path): @@ -4990,6 +5044,7 @@ class ProxyConfig: else MappingProxyType({}) ) changed_keys, removed_keys = changed_section_keys(baseline_section, new_section) + self.reject_config_owned_writes(section_name=section_name, changed_keys=changed_keys) if not changed_keys and not removed_keys: return wrote_section: Final = await self._upsert_changed_config_section( @@ -4998,10 +5053,38 @@ class ProxyConfig: removed_keys=removed_keys, prisma_client=prisma_client, ) - if not wrote_section: + if wrote_section is None: return + store: Final = self._settings_stores.get(cast(Section, section_name)) + if store is not None: + store.apply_db_row(cast(DbRow, section_name), wrote_section) await invalidate_config_param(section_name) + def reject_config_owned_writes(self, *, section_name: str, changed_keys: Mapping[str, JsonValue]) -> None: + """Refuse a write to a setting the config file owns, rather than storing a value that never applies.""" + store: Final = self._settings_stores.get(cast(Section, section_name)) + if store is None: + return + rejected: Final = store.rejected_writes(changed_keys) + if not rejected: + return + subject: Final = ( + f"key '{rejected[0]}' is" if len(rejected) == 1 else f"keys {', '.join(repr(key) for key in rejected)} are" + ) + pronoun: Final = "it" if len(rejected) == 1 else "them" + raise HTTPException( + status_code=400, + detail={ + "error": f"{section_name} {subject} set in the config file and cannot be changed here", + "keys": list(rejected), + "section": section_name, + "resolution": ( + f"edit {user_config_file_path} to change {pronoun}, " + f"or remove {pronoun} from the file to let the database own {pronoun}" + ), + }, + ) + async def _upsert_changed_config_section( self, *, @@ -5009,7 +5092,7 @@ class ProxyConfig: changed_keys: Mapping[str, JsonValue], removed_keys: frozenset[str], prisma_client: PrismaClient, - ) -> bool: + ) -> Mapping[str, JsonValue] | None: async with prisma_client.tx() as tx: await tx.query_raw(_CONFIG_SECTION_LOCK_SQL, section_name) config_table: Final = cast("TableActions[_ConfigParamRow]", tx.litellm_config) @@ -5033,14 +5116,14 @@ class ProxyConfig: } ) if merged_section == existing_section: - return False + return None serialized_section: Final = json.dumps(dict(merged_section)) # mutable-ok: JSON encoder requires a dict config_data: Final[_ConfigParamUpsert] = { "create": {"param_name": section_name, "param_value": serialized_section}, "update": {"param_value": serialized_section}, } await config_table.upsert(where=config_where, data=config_data) - return True + return merged_section async def save_environment_variables(self, updates: dict[str, str | None]) -> None: """Persist specific environment variables to the DB config row. @@ -5374,6 +5457,8 @@ class ProxyConfig: config = await self._get_config_from_file(config_file_path=config_file_path) + self._load_yaml_settings_stores(config) + ## UPDATE CONFIG WITH DB if prisma_client is not None and store_model_in_db is True: config = await self._update_config_from_db( @@ -5382,6 +5467,8 @@ class ProxyConfig: store_model_in_db=store_model_in_db, ) + config = self._config_with_resolved_settings(config) + ## PRINT YAML FOR CONFIRMING IT WORKS printed_yaml: Final = copy.deepcopy(config) printed_yaml.pop("environment_variables", None) @@ -5389,6 +5476,7 @@ class ProxyConfig: self._initialize_secret_manager_from_raw_config(config=config, config_file_path=config_file_path) config = self._check_for_os_environ_vars(config=config) + self._apply_resolved_runtime_settings(config) self.update_config_state(config=config) @@ -5967,17 +6055,6 @@ class ProxyConfig: _hc_staleness = None _hc_ignore_transient = False if general_settings: - # Record which keys were explicitly set in the YAML config file. - # These keys take precedence over DB-cached values during periodic - # reloads (see _update_general_settings). - self._yaml_general_settings_keys = set(general_settings.keys()) # mutable-ok: snapshot of YAML keys at load time # fmt: skip - # The VALUES matter for the cleanup bounds, not just which keys were - # set: clearing one from the dashboard has to fall back to what the - # YAML declared, and a set of names cannot answer that. - self._yaml_spend_log_cleanup_bounds = { # mutable-ok: snapshot of YAML bounds at load time # fmt: skip - key: general_settings[key] for key in SPEND_LOG_CLEANUP_BOUND_SETTINGS if key in general_settings - } - ### LOAD KEY MANAGEMENT SETTINGS ### # The secret manager itself is brought up by get_config(), which runs before the # `os.environ/` references in this config were resolved. Re-reading the settings here @@ -6123,7 +6200,6 @@ class ProxyConfig: ## pass through endpoints if general_settings.get("pass_through_endpoints", None) is not None: - config_passthrough_endpoints = general_settings["pass_through_endpoints"] await initialize_pass_through_endpoints( pass_through_endpoints=general_settings["pass_through_endpoints"], config_file_path=config_file_path, @@ -6164,13 +6240,6 @@ class ProxyConfig: health_check_interval = general_settings.get("health_check_interval", DEFAULT_HEALTH_CHECK_INTERVAL) health_check_concurrency = general_settings.get("health_check_concurrency", None) health_check_details = general_settings.get("health_check_details", True) - ### INTERACTIONS API SCHEMA ### - _use_legacy_interactions_schema: Final = general_settings.get("use_legacy_interactions_schema") - if _use_legacy_interactions_schema is not None: - if isinstance(_use_legacy_interactions_schema, str): - litellm.use_legacy_interactions_schema = _use_legacy_interactions_schema.lower() == "true" - else: - litellm.use_legacy_interactions_schema = bool(_use_legacy_interactions_schema) # Health-check-driven routing (opt-in, passes through to Router later) _enable_hc_routing = general_settings.get("enable_health_check_routing", False) _hc_staleness = general_settings.get("health_check_staleness_threshold", None) @@ -6357,7 +6426,8 @@ class ProxyConfig: ## NON-LLM CONFIGS eg. MCP tools, vector stores, etc. await self._init_non_llm_configs(config=config, config_file_path=config_file_path) - return router, router.get_model_list(), general_settings + _bind_general_settings_store(self.settings) + return router, router.get_model_list(), self.settings async def _init_non_llm_configs(self, config: dict, config_file_path: str | None = None): """ @@ -6805,13 +6875,6 @@ class ProxyConfig: config_data=config_data, llm_router=llm_router, prisma_client=prisma_client ) - # general settings - self._add_general_settings_from_db_config( - config_data=config_data, - general_settings=general_settings, - proxy_logging_obj=proxy_logging_obj, - ) - return still_desired_ids def _add_callback_from_db_to_in_memory_litellm_callbacks( @@ -7018,49 +7081,25 @@ class ProxyConfig: async def _add_router_settings_from_db_config( self, - config_data: dict, + config_data: Mapping[str, object], llm_router: Router | None, prisma_client: PrismaClient | None, ) -> None: - """ - Adds router settings from DB config to litellm proxy - - 1. Get router settings from DB - 2. Get router settings from config - 3. Combine both - 4. Update router settings - """ - if llm_router is not None and prisma_client is not None: - db_router_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first( - where={"param_name": "router_settings"} - ) - - config_router_settings: Final = config_data.get("router_settings", {}) - - combined_router_settings = {} - if ( - config_router_settings is not None - and isinstance(config_router_settings, dict) - and db_router_settings is not None - and isinstance(db_router_settings.param_value, dict) - ): - from litellm.utils import _update_dictionary - - db_overlay_deferring_empty_lists_to_config: Final = { - k: v - for k, v in db_router_settings.param_value.items() - if not (k in config_router_settings and isinstance(v, list) and len(v) == 0) - } - combined_router_settings = _update_dictionary( - config_router_settings, db_overlay_deferring_empty_lists_to_config - ) - elif config_router_settings is not None and isinstance(config_router_settings, dict): - combined_router_settings = config_router_settings - elif db_router_settings is not None and isinstance(db_router_settings.param_value, dict): - combined_router_settings = db_router_settings.param_value - - if combined_router_settings: - self._apply_router_settings(llm_router, combined_router_settings) + if llm_router is None or prisma_client is None: + return + self.router_settings.load_yaml(_as_settings_mapping(config_data.get("router_settings"))) + db_router_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first( + where={"param_name": "router_settings"} + ) + db_values: Final = ( + _as_settings_mapping(db_router_settings.param_value) + if db_router_settings is not None and db_router_settings.param_value is not None + else _EMPTY_SETTINGS_MAPPING + ) + self.router_settings.apply_db_row("router_settings", db_values) + combined_router_settings: Final = self.router_settings.resolved() + if combined_router_settings: + self._apply_router_settings(llm_router, combined_router_settings) @staticmethod def _apply_router_settings(llm_router: Router, router_settings: Mapping[str, object]) -> None: @@ -7222,260 +7261,143 @@ class ProxyConfig: except ValueError: verbose_proxy_logger.error("Invalid maximum_spend_logs_retention_interval value") - async def _update_general_settings(self, db_general_settings: Json | None): - """ - Pull from DB, read general settings value - """ - global general_settings, store_model_in_db + async def _update_general_settings(self, db_general_settings: Mapping[str, SettingsJsonValue] | None) -> None: + global general_settings if db_general_settings is None: return - _general_settings: Final = dict(db_general_settings) - ## MAX PARALLEL REQUESTS ## - if "max_parallel_requests" in _general_settings: - general_settings["max_parallel_requests"] = _general_settings["max_parallel_requests"] + if not isinstance(general_settings, SettingsStore): + self.settings.load_yaml(_as_settings_mapping(general_settings)) + cache_size_was_db: Final = self.settings.source("user_api_key_cache_max_size") == "db" + previous_retention_values: Final = self._resolved_retention_values() + previous_pass_through_endpoints: Final = self.settings.get("pass_through_endpoints") + self.settings.apply_db_row("general_settings", db_general_settings) + _bind_general_settings_store(self.settings) + await self._apply_general_settings_side_effects( + db_general_settings, + cache_size_was_db, + previous_retention_values, + previous_pass_through_endpoints, + ) - if "global_max_parallel_requests" in _general_settings: - general_settings["global_max_parallel_requests"] = _general_settings["global_max_parallel_requests"] - - if "max_batch_file_size_mb" not in self._yaml_general_settings_keys: - general_settings["max_batch_file_size_mb"] = _general_settings.get("max_batch_file_size_mb") - - if "max_file_size_mb" not in self._yaml_general_settings_keys: - general_settings["max_file_size_mb"] = _general_settings.get("max_file_size_mb") - - if "allowed_file_extensions" not in self._yaml_general_settings_keys: - general_settings["allowed_file_extensions"] = _general_settings.get("allowed_file_extensions") - - if "blocked_file_extensions" not in self._yaml_general_settings_keys: - general_settings["blocked_file_extensions"] = _general_settings.get("blocked_file_extensions") - - ## ALERTING ARGS ## - if "alerting_args" in _general_settings: - general_settings["alerting_args"] = _general_settings["alerting_args"] - proxy_logging_obj.slack_alerting_instance.update_values( - alerting_args=general_settings["alerting_args"], + def _resolved_retention_values(self) -> tuple[SettingsJsonValue | None, ...]: + return tuple( + self.settings.get(key) + for key in ( + "maximum_spend_logs_retention_period", + "maximum_autorouter_session_retention_period", + "maximum_health_check_retention_period", ) + ) - ## PASS-THROUGH ENDPOINTS ## - if "pass_through_endpoints" in _general_settings: - db_pass_through_endpoints: Final = _general_settings["pass_through_endpoints"] - db_pass_through_paths: Final = frozenset( - endpoint.get("path") for endpoint in db_pass_through_endpoints if isinstance(endpoint, dict) - ) - general_settings["pass_through_endpoints"] = [ - *db_pass_through_endpoints, - *( - endpoint - for endpoint in config_passthrough_endpoints or () - if endpoint.get("path") not in db_pass_through_paths - ), - ] - await initialize_pass_through_endpoints(pass_through_endpoints=db_pass_through_endpoints) - - ## UI ACCESS MODE ## - if "ui_access_mode" in _general_settings: - general_settings["ui_access_mode"] = _general_settings["ui_access_mode"] - - ## STORE PROMPTS IN SPEND LOGS ## - if "store_prompts_in_spend_logs" in _general_settings: - # If the YAML config explicitly set this key, prefer the YAML value - # over the DB-cached value. This ensures config changes deployed via - # CI/CD take effect without requiring a manual /config/update call. - # When YAML does not set this key, the DB value is used (preserving - # admin UI runtime changes). - if "store_prompts_in_spend_logs" in self._yaml_general_settings_keys: - value = general_settings.get("store_prompts_in_spend_logs") - else: - value = _general_settings["store_prompts_in_spend_logs"] - # Normalize case: handle True/true/TRUE, False/false/FALSE, None/null - if value is None: - general_settings["store_prompts_in_spend_logs"] = None - elif isinstance(value, bool): - general_settings["store_prompts_in_spend_logs"] = value - elif isinstance(value, str): - # Case-insensitive string comparison - general_settings["store_prompts_in_spend_logs"] = value.lower() == "true" - else: - # For other types, convert to bool - general_settings["store_prompts_in_spend_logs"] = bool(value) - - if "disable_auto_add_proxy_admin_to_teams" in _general_settings: - value = _general_settings["disable_auto_add_proxy_admin_to_teams"] - if isinstance(value, str): - general_settings["disable_auto_add_proxy_admin_to_teams"] = value.lower() == "true" - else: - general_settings["disable_auto_add_proxy_admin_to_teams"] = value if value is None else bool(value) - - if "apply_user_budget_to_team_keys" in _general_settings and ( - "apply_user_budget_to_team_keys" not in self._yaml_general_settings_keys - ): - db_value: Final = _general_settings["apply_user_budget_to_team_keys"] - if isinstance(db_value, str): - general_settings["apply_user_budget_to_team_keys"] = db_value.lower() == "true" - else: - general_settings["apply_user_budget_to_team_keys"] = db_value if db_value is None else bool(db_value) - - if "enable_openai_websocket_passthrough" not in self._yaml_general_settings_keys: - general_settings["enable_openai_websocket_passthrough"] = _general_settings.get( - "enable_openai_websocket_passthrough" - ) - - if "user_api_key_cache_max_size" not in self._yaml_general_settings_keys: - db_cache_max_size: Final = _general_settings.get("user_api_key_cache_max_size") - try: - cache_max_size: Final = ConfigGeneralSettings.model_validate( - MappingProxyType({"user_api_key_cache_max_size": db_cache_max_size}) - ).user_api_key_cache_max_size - except ValidationError: - verbose_proxy_logger.warning( - "Ignoring invalid general_settings.user_api_key_cache_max_size=%r from the DB", db_cache_max_size - ) - else: - if cache_max_size is None: - general_settings.pop("user_api_key_cache_max_size", None) - else: - general_settings["user_api_key_cache_max_size"] = cache_max_size - user_api_key_cache.update_in_memory_max_size(cache_max_size) - - ## STORE MODEL IN DB ## - if "store_model_in_db" in _general_settings: - value = _general_settings["store_model_in_db"] - if value is None: - pass # Don't change store_model_in_db to None; keep current value - elif isinstance(value, bool): - store_model_in_db = value - elif isinstance(value, str): - store_model_in_db = value.lower() == "true" - else: - store_model_in_db = bool(value) - general_settings["store_model_in_db"] = store_model_in_db - - ## MAXIMUM SPEND LOGS RETENTION PERIOD ## - if "maximum_spend_logs_retention_period" in _general_settings: - old_value: Final = general_settings.get("maximum_spend_logs_retention_period") - new_value: Final = _general_settings["maximum_spend_logs_retention_period"] - general_settings["maximum_spend_logs_retention_period"] = new_value - # Reschedule cleanup job if value changed (including when set to None) - if old_value != new_value: - await self._reschedule_spend_log_cleanup_job() - - if "maximum_autorouter_session_retention_period" in _general_settings: - old_session_value: Final = general_settings.get("maximum_autorouter_session_retention_period") - new_session_value: Final = _general_settings["maximum_autorouter_session_retention_period"] - general_settings["maximum_autorouter_session_retention_period"] = new_session_value - if old_session_value != new_session_value: - await self._reschedule_spend_log_cleanup_job() - - if "maximum_health_check_retention_period" in _general_settings: - old_health_check_value: Final = general_settings.get("maximum_health_check_retention_period") - new_health_check_value: Final = _general_settings["maximum_health_check_retention_period"] - general_settings["maximum_health_check_retention_period"] = new_health_check_value - if old_health_check_value != new_health_check_value: - await self._reschedule_spend_log_cleanup_job() - - ## SPEND LOG CLEANUP BOUNDS ## - # The dashboard writes these straight to the DB, so without copying them - # here the running cleanup job never sees them. A key the DB no longer - # carries was cleared from the dashboard, and falls back to whatever - # config.yaml declared, or to None (the shipped default) when it declared - # nothing. Leaving the deleted DB value in memory would keep enforcing the - # bound the operator just removed. - for cleanup_key in SPEND_LOG_CLEANUP_BOUND_SETTINGS: - general_settings[cleanup_key] = _general_settings.get( - cleanup_key, self._yaml_spend_log_cleanup_bounds.get(cleanup_key) - ) - - for key in ( - "user_url_allowed_hosts", - "user_url_validation", - "provider_url_destination_allowed_hosts", - ): - if key in _general_settings: - general_settings[key] = _general_settings[key] - _apply_ssrf_general_settings(_general_settings) - - def _update_config_fields( + async def _apply_general_settings_side_effects( self, - current_config: dict, - param_name: Literal[ - "general_settings", - "router_settings", - "litellm_settings", - "environment_variables", - ], - db_param_value: Any, - ) -> dict: - """ - Updates the config fields with the new values from the DB + db_values: Mapping[str, SettingsJsonValue], + cache_size_was_db: bool, + previous_retention_values: tuple[SettingsJsonValue | None, ...], + previous_pass_through_endpoints: SettingsJsonValue | None, + ) -> None: + effects: Final = ( + self._apply_alerting_settings, + partial(self._apply_pass_through_settings, previous_endpoints=previous_pass_through_endpoints), + self._apply_boolean_settings, + partial(self._apply_cache_size_setting, cache_size_was_db=cache_size_was_db), + self._apply_store_model_in_db_setting, + partial(self._apply_retention_settings, previous_retention_values=previous_retention_values), + self._apply_ssrf_settings, + ) + for effect in effects: + await effect(db_values) - Args: - current_config (dict): Current configuration dictionary to update - param_name (Literal): Name of the parameter to update - db_param_value (Any): New value from the database + async def _apply_alerting_settings(self, db_values: Mapping[str, SettingsJsonValue]) -> None: + alerting: Final = self.settings.get("alerting") + if "alerting" in db_values and isinstance(alerting, list): + proxy_logging_obj.update_values(alerting=alerting) - Returns: - dict: Updated configuration dictionary - """ + alerting_args: Final = self.settings.get("alerting_args") + if "alerting_args" in db_values and self.settings.source("alerting_args") == "db": + proxy_logging_obj.slack_alerting_instance.update_values(alerting_args=alerting_args) - def _deep_merge_dicts(dst: dict, src: dict) -> None: - """ - Deep-merge src into dst, skipping None values and empty lists from src. - On conflicts, src (DB) wins, but empty lists are treated as "no value" and don't overwrite. - """ - stack: Final = [(dst, src)] - while stack: - d, s = stack.pop() - for k, v in s.items(): - if v is None: - # Preserve existing config when DB value is None (matches prior behavior) - continue - # Skip empty lists - treat them as "no value" to preserve file config - if isinstance(v, list) and len(v) == 0: - continue - if isinstance(v, dict) and isinstance(d.get(k), dict): - stack.append((d[k], v)) - else: - d[k] = v + alert_types: Final = self.settings.get("alert_types") + if "alert_types" in db_values and self.settings.source("alert_types") == "db": + proxy_logging_obj.alert_types = alert_types + proxy_logging_obj.slack_alerting_instance.update_values(alert_types=alert_types, llm_router=llm_router) - # Strip remote-URL module loads from the DB-overlay before merge — - # the YAML-load callsites have ``config_file_path`` set, so a - # DB-sourced ``s3://`` value would otherwise reach - # ``_load_instance_from_remote_storage`` without going through - # the runtime gate. - db_param_value = _scrub_db_overlay_remote_module_loads(section=param_name, db_value=db_param_value) + webhook_url: Final = self.settings.get("alert_to_webhook_url") + if "alert_to_webhook_url" in db_values and self.settings.source("alert_to_webhook_url") == "db": + proxy_logging_obj.slack_alerting_instance.update_values( + alert_to_webhook_url=webhook_url, llm_router=llm_router + ) - if param_name == "environment_variables": - decrypted_env_vars = self._decrypt_and_set_db_env_variables(db_param_value, return_original_value=True) - # Normalize keys when loading from DB so services expecting uppercase - # (e.g. Datadog) can read them even if stored in lowercase. - merged_env_vars: Final[dict] = {} - for key, value in decrypted_env_vars.items(): - merged_env_vars[key] = value - upper_key = key.upper() - merged_env_vars[upper_key] = value - os.environ[upper_key] = value + if "plugins" in db_values and self.settings.source("plugins") == "db": + register_plugins_from_config(self.settings) - current_config.setdefault("environment_variables", {}).update(merged_env_vars) - return current_config - elif param_name == "litellm_settings" and isinstance(db_param_value, dict): - for key, value in db_param_value.items(): - if key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES: # params that are safe to override with db values - setattr(litellm, key, value) + async def _apply_pass_through_settings( + self, + db_values: Mapping[str, SettingsJsonValue], + previous_endpoints: SettingsJsonValue | None, + ) -> None: + del db_values + resolved_endpoints: Final = self.settings.get("pass_through_endpoints") + if resolved_endpoints == previous_endpoints: + return + await initialize_pass_through_endpoints( + pass_through_endpoints=resolved_endpoints if isinstance(resolved_endpoints, list) else [] + ) - # If param doesn't exist in config, add it - if param_name not in current_config: - current_config[param_name] = db_param_value + async def _apply_boolean_settings(self, db_values: Mapping[str, SettingsJsonValue]) -> None: + for key in ( + "store_prompts_in_spend_logs", + "disable_auto_add_proxy_admin_to_teams", + "apply_user_budget_to_team_keys", + ): + if key in db_values and (value := self.settings.get(key)) is not None: + self.settings[key] = coerce_bool(value) - return current_config - - # For dictionary values, update only non-none values - if isinstance(current_config[param_name], dict) and isinstance(db_param_value, dict): - _deep_merge_dicts(current_config[param_name], db_param_value) + async def _apply_cache_size_setting( + self, + db_values: Mapping[str, SettingsJsonValue], + cache_size_was_db: bool, + ) -> None: + if "user_api_key_cache_max_size" not in db_values and not cache_size_was_db: + return + cache_value: Final = self.settings.get("user_api_key_cache_max_size") + try: + cache_max_size: Final = ConfigGeneralSettings.model_validate( + MappingProxyType({"user_api_key_cache_max_size": cache_value}) + ).user_api_key_cache_max_size + except ValidationError: + self.settings.pop("user_api_key_cache_max_size", None) + verbose_proxy_logger.warning( + "Ignoring invalid general_settings.user_api_key_cache_max_size=%r from the DB", cache_value + ) + return + if cache_max_size is None: + self.settings.pop("user_api_key_cache_max_size", None) else: - # Non-dict or mismatched types: DB value replaces config (unchanged behavior) - current_config[param_name] = db_param_value + self.settings["user_api_key_cache_max_size"] = cache_max_size + user_api_key_cache.update_in_memory_max_size(cache_max_size) - return current_config + async def _apply_store_model_in_db_setting(self, db_values: Mapping[str, SettingsJsonValue]) -> None: + global store_model_in_db + if "store_model_in_db" not in db_values: + return + value: Final = self.settings.get("store_model_in_db") + if value is None: + return + normalized: Final = coerce_bool(value) + store_model_in_db = normalized if isinstance(normalized, bool) else bool(normalized) + self.settings["store_model_in_db"] = store_model_in_db + + async def _apply_retention_settings( + self, + db_values: Mapping[str, SettingsJsonValue], + previous_retention_values: tuple[SettingsJsonValue | None, ...], + ) -> None: + if previous_retention_values != self._resolved_retention_values(): + await self._reschedule_spend_log_cleanup_job() + + async def _apply_ssrf_settings(self, db_values: Mapping[str, SettingsJsonValue]) -> None: + _apply_ssrf_general_settings(db_values) async def _update_config_from_db( self, @@ -7487,37 +7409,48 @@ class ProxyConfig: verbose_proxy_logger.info("'store_model_in_db' is not True, skipping db updates") return config - _tasks: Final = [] - keys: Final = [ - "general_settings", - "router_settings", - "litellm_settings", - "environment_variables", - ] - for k in keys: - _tasks.append(get_config_param(prisma_client, k)) - - responses: Final = await asyncio.gather(*_tasks) - for response in responses: - if response is None: + sections: Final = tuple(self._settings_stores) + responses: Final = await asyncio.gather(*(get_config_param(prisma_client, section) for section in sections)) + for section, response in zip(sections, responses): + if response is None or (param_value := getattr(response, "param_value", None)) is None: continue - param_name = getattr(response, "param_name", None) - param_value = getattr(response, "param_value", None) verbose_proxy_logger.debug( "param_name=%s, param_value=%s", - param_name, - _redact_config_param_value_for_logging(param_name, param_value), + section, + _redact_config_param_value_for_logging(section, param_value), ) - - if param_name is not None and param_value is not None: - config = self._update_config_fields( - current_config=config, - param_name=param_name, - db_param_value=param_value, + if section == "litellm_settings": + self._apply_litellm_settings_db_values(self._prepared_db_settings_values(section, param_value)) + else: + self._settings_stores[section].apply_db_row( + section, + self._prepared_db_settings_values(section, param_value), ) - return config + return self._config_with_resolved_settings(config) + + def _prepared_db_settings_values(self, section: Section, value: object) -> Mapping[str, SettingsJsonValue]: + if section == "environment_variables": + decrypted: Final = self._decrypt_and_set_db_env_variables( + dict(_as_settings_mapping(value)), return_original_value=True + ) + normalized: Final = { + **decrypted, + **{key.upper(): decrypted_value for key, decrypted_value in decrypted.items()}, + } + for key, decrypted_value in normalized.items(): + os.environ[key] = decrypted_value + return _as_settings_mapping(normalized) + + scrubbed: Final = _scrub_db_overlay_remote_module_loads(section=section, db_value=value) + return _as_settings_mapping(scrubbed) + + def _apply_litellm_settings_db_values(self, db_values: Mapping[str, SettingsJsonValue]) -> None: + self.litellm_settings.apply_db_row("litellm_settings", db_values) + for key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES: + if key in db_values and (value := self.litellm_settings.get(key)) is not None: + setattr(litellm, key, value) def _should_load_db_object(self, object_type: str | SupportedDBObjectType) -> bool: return should_load_db_object(object_type=object_type) @@ -7753,12 +7686,8 @@ class ProxyConfig: if config_record is None or config_record.param_value is None: return raw_settings: Final = config_record.param_value - litellm_settings: Final = json.loads(raw_settings) if isinstance(raw_settings, str) else raw_settings - if not isinstance(litellm_settings, dict): - return - for key, value in litellm_settings.items(): - if key in LITELLM_SETTINGS_SAFE_DB_OVERRIDES: - setattr(litellm, key, value) + db_values: Final = self._prepared_db_settings_values("litellm_settings", raw_settings) + self._apply_litellm_settings_db_values(db_values) async def _init_semantic_filter_settings_in_db(self, prisma_client: PrismaClient): """ @@ -17406,6 +17335,11 @@ async def update_config_general_settings( ## update db + proxy_config.reject_config_owned_writes( + section_name="general_settings", + changed_keys={data.field_name: cast(JsonValue, data.field_value)}, # cast-ok: validated above + ) + field_value = data.field_value if data.field_name == "plugins": field_value = _preserve_redacted_plugin_keys(field_value, general_settings.get("plugins")) @@ -17423,6 +17357,7 @@ async def update_config_general_settings( }, ) await invalidate_config_param("general_settings") + proxy_config.settings.apply_db_row("general_settings", general_settings) asyncio.create_task( create_config_audit_log( "general_settings", "updated", before_general_settings, general_settings, user_api_key_dict @@ -17611,37 +17546,30 @@ async def get_config_general_settings( detail={"error": f"Invalid field={field_name} passed in."}, ) - ## get general settings from db - db_general_settings: Final[_ConfigParamRow | None] = await _config_param_table(prisma_client).find_first( - where={"param_name": "general_settings"} - ) - ### pop the value - - if db_general_settings is None or db_general_settings.param_value is None: + settings: Final = proxy_config.settings + if field_name not in settings: raise HTTPException( status_code=400, - detail={"error": f"Field name={field_name} not in DB"}, + detail={"error": f"Field name={field_name} is not set"}, ) - else: - general_settings = dict(db_general_settings.param_value) - if field_name in general_settings: - field_value = _redact_general_setting_value( - field_name, - general_settings[field_name], - user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN, - ) - if field_name == "plugins" and isinstance(field_value, list): - field_value = [ - ({k: ("***" if k == "plugin_key" else v) for k, v in p.items()} if isinstance(p, dict) else p) - for p in field_value - ] - return ConfigFieldInfo(field_name=field_name, field_value=field_value) - else: - raise HTTPException( - status_code=400, - detail={"error": f"Field name={field_name} not in DB"}, - ) + field_value = _redact_general_setting_value( + field_name, + settings[field_name], + user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN, + ) + if field_name == "plugins" and isinstance(field_value, list): + field_value = [ + ({k: ("***" if k == "plugin_key" else v) for k, v in p.items()} if isinstance(p, dict) else p) + for p in field_value + ] + source: Final = settings.source(field_name) + return ConfigFieldInfo( + field_name=field_name, + field_value=field_value, + source=source, + editable=source != "config", + ) GeneralSettingsUILiteLLMValue = float | bool | str | None @@ -17759,6 +17687,7 @@ async def _persist_general_settings_ui_litellm_field( field_name: str, value: object, user_api_key_dict: UserAPIKeyAuth ) -> dict: validated: Final = _validate_general_settings_ui_litellm_value(field_name, value) + proxy_config.reject_config_owned_writes(section_name="litellm_settings", changed_keys={field_name: validated}) config: Final = await proxy_config.get_config() before_value: Final = config.get("litellm_settings", {}).get(field_name) setattr(litellm, field_name, validated) @@ -17771,9 +17700,10 @@ async def _persist_general_settings_ui_litellm_field( async def _reset_general_settings_ui_litellm_field(field_name: str, user_api_key_dict: UserAPIKeyAuth) -> dict: + default_value: Final = _general_settings_ui_litellm_default(_GENERAL_SETTINGS_UI_LITELLM_FIELDS[field_name]) + proxy_config.reject_config_owned_writes(section_name="litellm_settings", changed_keys={field_name: default_value}) config: Final = await proxy_config.get_config() before_value: Final = config.get("litellm_settings", {}).get(field_name) - default_value: Final = _general_settings_ui_litellm_default(_GENERAL_SETTINGS_UI_LITELLM_FIELDS[field_name]) setattr(litellm, field_name, default_value) if "litellm_settings" in config: config["litellm_settings"].pop(field_name, None) @@ -17874,6 +17804,7 @@ async def get_config_list( _stored_in_db = True elif field_name in general_settings: _stored_in_db = False + _source = proxy_config.settings.source(field_name) _response_obj = ConfigList( field_name=field_name, @@ -17887,6 +17818,8 @@ async def get_config_list( stored_in_db=_stored_in_db, field_default_value=field_info.default, nested_fields=nested_fields, + source=_source, + editable=_source != "config", ) return_val.append(_response_obj) @@ -17899,8 +17832,9 @@ async def get_config_list( elif field_name in general_settings: _stored_in_db = False + _source = proxy_config.settings.source(field_name) _field_value = general_settings.get(field_name, None) - if _field_value is None and field_name in db_general_settings_dict: + if _field_value is None and _source != "config" and field_name in db_general_settings_dict: _field_value = db_general_settings_dict[field_name] _response_obj = ConfigList( @@ -17911,6 +17845,8 @@ async def get_config_list( stored_in_db=_stored_in_db, field_default_value=field_info.default, nested_fields=nested_fields, + source=_source, + editable=_source != "config", ) return_val.append(_response_obj) @@ -17932,6 +17868,7 @@ async def get_config_list( stored_in_db_litellm = False else: stored_in_db_litellm = None + _litellm_source = proxy_config.litellm_settings.source(litellm_field_name) return_val.append( ConfigList( field_name=litellm_field_name, @@ -17943,6 +17880,8 @@ async def get_config_list( field_options=list(spec.get("options", ())) or None, field_tab=spec.get("tab"), nested_fields=None, + source=_litellm_source, + editable=_litellm_source != "config", ) ) @@ -18021,6 +17960,7 @@ async def delete_config_general_settings( }, ) await invalidate_config_param("general_settings") + proxy_config.settings.apply_db_row("general_settings", general_settings) asyncio.create_task( create_config_audit_log( "general_settings", "deleted", before_general_settings, general_settings, user_api_key_dict diff --git a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py index de965aff889..fd160636d46 100644 --- a/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py @@ -1483,10 +1483,13 @@ _UI_SETTINGS_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) def apply_runtime_general_settings_flags(ui_settings: Mapping[str, JsonValue]) -> Mapping[str, JsonValue]: """Copy the UI settings that gate runtime behavior into ``general_settings``. Returns what was applied.""" + from litellm.proxy.config_resolvers import SettingsStore from litellm.proxy.proxy_server import general_settings flags: Final = {k: ui_settings[k] for k in _RUNTIME_GENERAL_SETTINGS_FLAGS if k in ui_settings} - if flags: + if isinstance(general_settings, SettingsStore): + general_settings.apply_db_row("ui_settings", flags) + elif flags: general_settings.update(flags) return MappingProxyType(flags) diff --git a/litellm/repositories/config_repository.py b/litellm/repositories/config_repository.py index 2e8e760db07..8b8280622fd 100644 --- a/litellm/repositories/config_repository.py +++ b/litellm/repositories/config_repository.py @@ -1,25 +1,18 @@ -""" -Config repository for database operations on LiteLLM_Config. +"""Config repository for database operations on LiteLLM_Config.""" -This repository handles config reconciliation between database values and -YAML configmap values. DB values override configmap values except for -None values and empty lists. -""" +from __future__ import annotations -import asyncio -import copy import json -import os from collections.abc import Mapping, Sequence -from typing import Any, Final, Literal, Protocol, cast +from typing import TYPE_CHECKING, Final, Protocol, cast -from litellm._logging import verbose_proxy_logger -from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper +if TYPE_CHECKING: + from litellm.proxy.utils import PrismaClient def _decoded_json(raw: str) -> object: """Decode a JSON-encoded config row value into an opaque object.""" - return json.loads(raw) + return cast(object, json.loads(raw)) class _ConfigRow(Protocol): @@ -40,16 +33,6 @@ class _ConfigTable(Protocol): async def delete(self, *, where: Mapping[str, str]) -> _ConfigRow | None: ... -class _ConfigDb(Protocol): - @property - def litellm_config(self) -> _ConfigTable: ... - - -class _PrismaHandle(Protocol): - @property - def db(self) -> _ConfigDb: ... - - class ConfigParam: """Simple wrapper for config parameter from DB.""" @@ -59,27 +42,20 @@ class ConfigParam: class ConfigRepository: - """Repository for config database operations with reconciliation support.""" + """Repository for config database operations.""" - CONFIG_PARAMS = [ - "general_settings", - "router_settings", - "litellm_settings", - "environment_variables", - ] - - def __init__(self, prisma_client: Any): - self._prisma_client = prisma_client + def __init__(self, prisma_client: PrismaClient | None): + self._prisma_client: Final = prisma_client @property - def prisma_client(self) -> _PrismaHandle: + def prisma_client(self) -> PrismaClient: if self._prisma_client is None: raise RuntimeError("No DB Connected. See - https://docs.litellm.ai/docs/proxy/virtual_keys") return self._prisma_client @property def _config_table(self) -> _ConfigTable: - return self.prisma_client.db.litellm_config + return cast(_ConfigTable, self.prisma_client.db.litellm_config) @property def table(self) -> _ConfigTable: @@ -125,141 +101,3 @@ class ConfigRepository: param_value = _decoded_json(param_value) result[record.param_name] = param_value return result - - def _deep_merge_dicts(self, dst: dict, src: dict) -> None: - """Deep-merge src into dst, skipping None values and empty lists from src. - - On conflicts, src (DB) wins, but empty lists are treated as "no value" - and don't overwrite the destination. - """ - stack: Final = [(dst, src)] - while stack: - d, s = stack.pop() - for k, v in s.items(): - if v is None: - continue - if isinstance(v, list) and len(v) == 0: - continue - if isinstance(v, dict) and isinstance(d.get(k), dict): - stack.append((d[k], v)) - else: - d[k] = v - - def _decrypt_env_variables( - self, env_vars: Mapping[str, object], return_original_value: bool = True - ) -> dict[str, str]: - """Decrypt environment variables from database.""" - decrypted: Final[dict[str, str]] = {} - for key, value in env_vars.items(): - if isinstance(value, str): - decrypted_value = decrypt_value_helper( - value=value, - key=key, - exception_type="debug", - return_original_value=return_original_value, - ) - if decrypted_value is not None: - decrypted[key] = decrypted_value - else: - decrypted[key] = str(value) - return decrypted - - def _normalize_env_variable_keys(self, env_vars: dict[str, str]) -> dict[str, str]: - """Normalize env variable keys to include both original and uppercase versions.""" - normalized: Final[dict[str, str]] = {} - for key, value in env_vars.items(): - normalized[key] = value - upper_key = key.upper() - normalized[upper_key] = value - return normalized - - def _update_config_fields( - self, - current_config: dict, - param_name: Literal[ - "general_settings", - "router_settings", - "litellm_settings", - "environment_variables", - ], - db_param_value: Any, - ) -> dict: - """Update config fields with DB values, handling the merge strategy.""" - if param_name == "environment_variables": - decrypted_env_vars: Final = self._decrypt_env_variables(db_param_value, return_original_value=True) - merged_env_vars: Final = self._normalize_env_variable_keys(decrypted_env_vars) - for env_key, value in merged_env_vars.items(): - os.environ[env_key] = value - - current_config.setdefault("environment_variables", {}).update(merged_env_vars) - return current_config - - if param_name not in current_config: - current_config[param_name] = db_param_value - return current_config - - if isinstance(current_config[param_name], dict) and isinstance(db_param_value, dict): - self._deep_merge_dicts(current_config[param_name], db_param_value) - else: - current_config[param_name] = db_param_value - - return current_config - - async def reconcile_config( - self, - yaml_config: dict, - store_model_in_db: bool | None = None, - ) -> dict: - """Reconcile config from YAML with database overrides. - - This is the main config reconciliation method that loads config params - from the database and merges them with the YAML config. DB values - override YAML values except for None values and empty lists. - - Args: - yaml_config: The configuration loaded from YAML file - store_model_in_db: Whether to load config from DB - - Returns: - The merged configuration with DB overrides applied - """ - if store_model_in_db is not True: - verbose_proxy_logger.info("'store_model_in_db' is not True, skipping db config reconciliation") - return yaml_config - - tasks: Final = [self.get_param(k) for k in self.CONFIG_PARAMS] - responses: Final = await asyncio.gather(*tasks) - - config = copy.deepcopy(yaml_config) - for response in responses: - if response is None: - continue - - param_name = response.param_name - param_value = response.param_value - verbose_proxy_logger.debug("param_name=%s, param_value=%s", param_name, param_value) - - if param_name is not None and param_value is not None: - config = self._update_config_fields( - current_config=config, - param_name=cast( - Literal[ - "general_settings", - "router_settings", - "litellm_settings", - "environment_variables", - ], - param_name, - ), - db_param_value=param_value, - ) - - return config - - async def prefetch_params(self, param_names: list[str]) -> None: - """Prefetch config params to warm the cache. - - This can be called before reconcile_config to ensure all needed - params are loaded in a single batch. - """ - await asyncio.gather(*[self.get_param(k) for k in param_names]) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 7191a33a74a..dc1121977ec 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -41100,21 +41100,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro": { - "input_cost_per_token": 9.4336e-07, + "input_cost_per_token": 1.6e-06, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 1.88672e-06, + "output_cost_per_token": 3.2e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 7.9596e-08, + "cache_read_input_token_cost": 1.35e-07, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -41141,21 +41141,21 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-pro-0813": { - "input_cost_per_token": 6.6e-07, + "input_cost_per_token": 5.7816e-07, "input_cost_per_token_cache_hit": 4.4e-08, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 1.98e-06, + "output_cost_per_token": 1.73448e-06, "source": "https://openrouter.ai/api/v1/models", "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, - "cache_read_input_token_cost": 2.2e-08, + "cache_read_input_token_cost": 1.8396e-08, "supports_audio_input": false, "supports_pdf_input": false, "supports_vision": false, @@ -41197,6 +41197,7 @@ "supports_vision": true, "supports_image_size": false, "cache_read_input_token_cost": 3e-08, + "deprecation_date": "2026-10-20", "supports_prompt_caching": true, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": true, @@ -41223,6 +41224,7 @@ "supports_vision": true, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "deprecation_date": "2026-10-20", "input_cost_per_token_above_200k_tokens": 2.5e-06, "output_cost_per_token_above_200k_tokens": 1.5e-05, "supports_prompt_caching": true, @@ -65084,6 +65086,7 @@ "supports_pdf_input": true, "supports_audio_input": true, "cache_read_input_token_cost": 1e-08, + "deprecation_date": "2026-10-20", "input_cost_per_audio_token": 3e-07, "supports_prompt_caching": true, "supports_web_search": false @@ -66130,9 +66133,9 @@ "supports_web_search": false }, "openrouter/z-ai/glm-5.2": { - "input_cost_per_token": 4.875e-07, - "output_cost_per_token": 1.56e-06, - "cache_read_input_token_cost": 9.1e-08, + "input_cost_per_token": 5.544e-07, + "output_cost_per_token": 1.7424e-06, + "cache_read_input_token_cost": 1.0296e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 131072, @@ -66492,9 +66495,9 @@ "supports_web_search": false }, "openrouter/deepseek/deepseek-v4-flash": { - "input_cost_per_token": 8.8606e-08, - "output_cost_per_token": 1.77212e-07, - "cache_read_input_token_cost": 1.77212e-08, + "input_cost_per_token": 4.984e-08, + "output_cost_per_token": 9.968e-08, + "cache_read_input_token_cost": 9.968e-09, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 384000, @@ -67162,6 +67165,7 @@ "cache_read_input_token_cost": 3e-08, "cache_creation_input_token_cost": 8.33333333333333e-08, "cache_read_input_audio_token_cost": 1e-07, + "deprecation_date": "2027-03-15", "input_cost_per_audio_token": 1e-06, "output_cost_per_image_token": 3e-05, "litellm_provider": "openrouter", @@ -67401,7 +67405,7 @@ "supports_audio_input": false, "supports_function_calling": true, "supports_pdf_input": false, - "supports_prompt_caching": true, + "supports_prompt_caching": false, "supports_reasoning": false, "supports_tool_choice": true, "supports_response_schema": true, @@ -70545,14 +70549,14 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-pro-latest": { - "cache_read_input_token_cost": 2.2e-08, - "input_cost_per_token": 6.6e-07, + "cache_read_input_token_cost": 1.8396e-08, + "input_cost_per_token": 5.7816e-07, "litellm_provider": "openrouter", "max_input_tokens": 1048576, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 1.98e-06, + "output_cost_per_token": 1.73448e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -70565,14 +70569,14 @@ "supports_web_search": false }, "openrouter/~deepseek/deepseek-v4-flash-latest": { - "cache_read_input_token_cost": 8.8e-09, - "input_cost_per_token": 5.58e-08, + "cache_read_input_token_cost": 1.75e-09, + "input_cost_per_token": 5.5e-08, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 393216, "max_tokens": 393216, "mode": "chat", - "output_cost_per_token": 1.767e-07, + "output_cost_per_token": 1.65e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -70817,14 +70821,14 @@ "supports_web_search": false }, "openrouter/~z-ai/glm-latest": { - "cache_read_input_token_cost": 1.755e-07, - "input_cost_per_token": 8.775e-07, + "cache_read_input_token_cost": 1.5e-07, + "input_cost_per_token": 9e-07, "litellm_provider": "openrouter", "max_input_tokens": 1310720, "max_output_tokens": 235929, "max_tokens": 235929, "mode": "chat", - "output_cost_per_token": 2.97e-06, + "output_cost_per_token": 3e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -71699,6 +71703,7 @@ "openrouter/google/gemini-2.5-flash:batch": { "cache_read_input_audio_token_cost": 1e-07, "cache_read_input_token_cost": 3e-08, + "deprecation_date": "2026-10-20", "input_cost_per_audio_token": 5e-07, "input_cost_per_token": 1.5e-07, "litellm_provider": "openrouter", @@ -71722,6 +71727,7 @@ "cache_read_input_audio_token_cost": 1.25e-07, "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_above_200k_tokens": 2.5e-07, + "deprecation_date": "2026-10-20", "input_cost_per_audio_token": 6.25e-07, "input_cost_per_token": 6.25e-07, "input_cost_per_token_above_200k_tokens": 1.25e-06, @@ -72237,13 +72243,13 @@ }, "openrouter/meta/muse-glimmer-30b": { "cache_read_input_token_cost": 4e-08, - "input_cost_per_token": 3e-07, + "input_cost_per_token": 3.5e-07, "litellm_provider": "openrouter", "max_input_tokens": 131072, "max_output_tokens": 117964, "max_tokens": 117964, "mode": "chat", - "output_cost_per_token": 1.1e-06, + "output_cost_per_token": 1.5e-06, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, diff --git a/tests/e2e/management/test_config_misc_endpoints_e2e.py b/tests/e2e/management/test_config_misc_endpoints_e2e.py index 0906ab52fe9..20e98e993d4 100644 --- a/tests/e2e/management/test_config_misc_endpoints_e2e.py +++ b/tests/e2e/management/test_config_misc_endpoints_e2e.py @@ -24,10 +24,10 @@ from collections.abc import Callable from typing import Final import pytest -from pydantic import BaseModel, JsonValue +from pydantic import BaseModel, JsonValue, RootModel from e2e_config import unique_marker -from e2e_http import NoBody, Success, UnknownApiError, unwrap, unwrap_status +from e2e_http import NoBody, Success, unwrap, unwrap_status from lifecycle import ResourceManager from management_client import ManagementClient from models import KeyGenerateBody, LiteLLMParamsBody, TeamNewBody @@ -210,6 +210,24 @@ class ConfigFieldInfoParams(BaseModel): class ConfigFieldInfoResponse(BaseModel): field_name: str field_value: JsonValue + source: str + editable: bool + + +class ConfigListParams(BaseModel): + config_type: str + + +class ConfigListEntry(BaseModel): + field_name: str + field_value: JsonValue + stored_in_db: bool | None + source: str + editable: bool + + +class ConfigListResponse(RootModel[list[ConfigListEntry]]): + pass class RouterCurrentValues(BaseModel): @@ -556,17 +574,30 @@ class TestConfigPersistence: ) assert added.message == f"IP {allowed_ip} address added successfully" - field_info: Final = client.proxy.transport.get( - "/config/field/info", - headers=client.proxy.transport.master, - params=ConfigFieldInfoParams(field_name="max_parallel_requests"), - response_type=ConfigFieldInfoResponse, + listed: Final = unwrap( + client.proxy.transport.get( + "/config/list", + headers=client.proxy.transport.master, + params=ConfigListParams(config_type="general_settings"), + response_type=ConfigListResponse, + ) ) - match field_info: - case UnknownApiError(status_code=400, body=body): - assert "not in DB" in body - case _: - pytest.fail(f"expected max_parallel_requests to remain absent from the DB row, got {field_info}") + unrelated: Final = next(entry for entry in listed.root if entry.field_name == "max_parallel_requests") + assert unrelated.stored_in_db is not True + assert unrelated.source == "config" + assert unrelated.editable is False + + field_info: Final = unwrap( + client.proxy.transport.get( + "/config/field/info", + headers=client.proxy.transport.master, + params=ConfigFieldInfoParams(field_name="max_parallel_requests"), + response_type=ConfigFieldInfoResponse, + ) + ) + assert field_info.source == "config" + assert field_info.editable is False + assert field_info.field_value == unrelated.field_value class TestMcpServerSubmission: diff --git a/tests/proxy_unit_tests/test_proxy_config_unit_test.py b/tests/proxy_unit_tests/test_proxy_config_unit_test.py index 81648dc1158..5f236806685 100644 --- a/tests/proxy_unit_tests/test_proxy_config_unit_test.py +++ b/tests/proxy_unit_tests/test_proxy_config_unit_test.py @@ -288,68 +288,55 @@ async def test_json_logs_calls_turn_on_json(): class TestYamlStorePromptsDbOverride: - """ - Test that YAML store_prompts_in_spend_logs takes precedence over DB-cached value. - - When store_model_in_db=true, LiteLLM persists general_settings to the DB. - On periodic reloads, _update_general_settings() must NOT override - YAML-explicit values with stale DB values. - """ - - def _make_proxy_config_with_yaml_keys(self, yaml_keys: set) -> "ProxyConfig": - """Helper: create ProxyConfig with pre-populated _yaml_general_settings_keys.""" - proxy_config = ProxyConfig() - proxy_config._yaml_general_settings_keys = yaml_keys - return proxy_config - @pytest.mark.asyncio async def test_yaml_value_takes_precedence_over_db(self): - """When YAML sets store_prompts_in_spend_logs=false, DB value (true) should be ignored.""" - proxy_config = self._make_proxy_config_with_yaml_keys({"store_prompts_in_spend_logs"}) + proxy_config = ProxyConfig() + proxy_config.settings.load_yaml({"store_prompts_in_spend_logs": False}) - test_general_settings = {"store_prompts_in_spend_logs": False} - - with mock.patch("litellm.proxy.proxy_server.general_settings", test_general_settings): + with mock.patch("litellm.proxy.proxy_server.general_settings", proxy_config.settings): await proxy_config._update_general_settings( db_general_settings={"store_prompts_in_spend_logs": True}, ) - assert test_general_settings["store_prompts_in_spend_logs"] is False + from litellm.proxy import proxy_server + + assert proxy_server.general_settings["store_prompts_in_spend_logs"] is False + assert proxy_server.general_settings.source("store_prompts_in_spend_logs") == "config" @pytest.mark.asyncio async def test_db_value_used_when_yaml_does_not_set_key(self): - """When YAML does NOT set store_prompts_in_spend_logs, DB value should be used.""" - proxy_config = self._make_proxy_config_with_yaml_keys({"master_key", "database_url"}) + proxy_config = ProxyConfig() + proxy_config.settings.load_yaml({"master_key": "sk-test"}) - test_general_settings = {"master_key": "sk-test"} - - with mock.patch("litellm.proxy.proxy_server.general_settings", test_general_settings): + with mock.patch("litellm.proxy.proxy_server.general_settings", proxy_config.settings): await proxy_config._update_general_settings( db_general_settings={"store_prompts_in_spend_logs": True}, ) - assert test_general_settings["store_prompts_in_spend_logs"] is True + from litellm.proxy import proxy_server + + assert proxy_server.general_settings["store_prompts_in_spend_logs"] is True + assert proxy_server.general_settings.source("store_prompts_in_spend_logs") == "db" @pytest.mark.asyncio async def test_admin_ui_change_works_when_yaml_omits_key(self): - """Admin UI change (DB update) should work when YAML doesn't set the key.""" - proxy_config = self._make_proxy_config_with_yaml_keys({"master_key"}) + proxy_config = ProxyConfig() + proxy_config.settings.load_yaml({"master_key": "sk-test"}) - test_general_settings = {"master_key": "sk-test"} - - with mock.patch("litellm.proxy.proxy_server.general_settings", test_general_settings): + with mock.patch("litellm.proxy.proxy_server.general_settings", proxy_config.settings): await proxy_config._update_general_settings( db_general_settings={"store_prompts_in_spend_logs": True}, ) - assert test_general_settings["store_prompts_in_spend_logs"] is True - await proxy_config._update_general_settings( db_general_settings={"store_prompts_in_spend_logs": False}, ) - assert test_general_settings["store_prompts_in_spend_logs"] is False + from litellm.proxy import proxy_server - def test_yaml_general_settings_keys_populated_on_load(self): - """_yaml_general_settings_keys should be empty on init.""" + assert proxy_server.general_settings["store_prompts_in_spend_logs"] is False + assert proxy_server.general_settings.source("store_prompts_in_spend_logs") == "db" + + def test_proxy_config_settings_start_unset(self): proxy_config = ProxyConfig() - assert proxy_config._yaml_general_settings_keys == set() + + assert proxy_config.settings.source("store_prompts_in_spend_logs") == "unset" diff --git a/tests/proxy_unit_tests/test_proxy_utils.py b/tests/proxy_unit_tests/test_proxy_utils.py index 35de9961054..160753e3442 100644 --- a/tests/proxy_unit_tests/test_proxy_utils.py +++ b/tests/proxy_unit_tests/test_proxy_utils.py @@ -699,18 +699,19 @@ async def test_proxy_config_update_from_db(): param_name: str param_value: dict - with patch.object( - pc, - "get_generic_data", - new=AsyncMock( - return_value=ReturnValue( - param_name="litellm_settings", - param_value={ - "success_callback": "langfuse", - }, - ) - ), - ): + async def get_litellm_settings(_: object, section: str) -> ReturnValue | None: + if section != "litellm_settings": + return None + return ReturnValue( + param_name="litellm_settings", + param_value={ + "success_callback": "langfuse", + }, + ) + + proxy_config._load_yaml_settings_stores(test_config) + + with patch("litellm.proxy.proxy_server.get_config_param", side_effect=get_litellm_settings): new_config = await proxy_config._update_config_from_db( prisma_client=pc, config=test_config, @@ -1090,7 +1091,7 @@ def test_get_team_models(): assert result == ["gpt-4o", "gpt-3.5-turbo", "gpt-4o-mini"] -def test_update_config_fields(): +def test_settings_store_preserves_yaml_team_configuration_when_db_value_is_null(): from litellm.proxy.proxy_server import ProxyConfig proxy_config = ProxyConfig() @@ -1120,13 +1121,10 @@ def test_update_config_fields(): "context_window_fallbacks": [{"gpt-3.5-turbo": ["gpt-3.5-turbo-large"]}], }, } - updated_config = proxy_config._update_config_fields(**args) + proxy_config.litellm_settings.load_yaml(args["current_config"]["litellm_settings"]) + proxy_config.litellm_settings.apply_db_row("litellm_settings", args["db_param_value"]) + all_team_config = proxy_config.litellm_settings["default_team_settings"] - print("updated_config", updated_config) - all_team_config = updated_config["litellm_settings"]["default_team_settings"] - - # check if team id config returned - print("all_team_config", all_team_config) team_config = proxy_config._get_team_config( team_id="c91e32bb-0f2a-4aa1-86c4-307ca2e03ea3", all_teams_config=all_team_config ) @@ -1135,7 +1133,7 @@ def test_update_config_fields(): assert team_config["langfuse_secret"] == "my-fake-secret" -def test_update_config_fields_default_internal_user_params(monkeypatch): +def test_settings_store_applies_default_internal_user_params_from_db(monkeypatch): from litellm.proxy.proxy_server import ProxyConfig proxy_config = ProxyConfig() @@ -1153,7 +1151,8 @@ def test_update_config_fields_default_internal_user_params(monkeypatch): }, }, } - proxy_config._update_config_fields(**args) + db_values = proxy_config._prepared_db_settings_values("litellm_settings", args["db_param_value"]) + proxy_config._apply_litellm_settings_db_values(db_values) assert litellm.default_internal_user_params == { "user_role": "proxy_admin", diff --git a/tests/test_litellm/interactions/test_gemini_interactions_transformation.py b/tests/test_litellm/interactions/test_gemini_interactions_transformation.py index d9b7cc790e6..2809c12ae47 100644 --- a/tests/test_litellm/interactions/test_gemini_interactions_transformation.py +++ b/tests/test_litellm/interactions/test_gemini_interactions_transformation.py @@ -2,7 +2,7 @@ Tests for Gemini Interactions API transformation. Covers: -- validate_environment: x-goog-api-key header, Api-Revision schema selection +- validate_environment: x-goog-api-key header, Api-Revision header - get_complete_url: API key excluded from URL - get/delete/cancel interaction request URLs - transform_request: response_mime_type coalescing, image_config migration @@ -13,7 +13,6 @@ from unittest.mock import MagicMock, patch import pytest -import litellm from litellm.interactions.litellm_responses_transformation.streaming_iterator import ( LiteLLMResponsesInteractionsStreamingIterator, ) @@ -83,22 +82,10 @@ class TestValidateEnvironment: assert headers["X-Custom"] == "value" assert headers["x-goog-api-key"] == "test-key" - def test_api_revision_new_schema_by_default(self, config, monkeypatch: pytest.MonkeyPatch): - # Default: use_legacy_interactions_schema=False → new steps schema - monkeypatch.setattr(litellm, "use_legacy_interactions_schema", False) - headers = config.validate_environment( - headers={}, model="gemini-2.5-flash", litellm_params=None - ) + def test_sets_api_revision_header(self, config): + headers = config.validate_environment(headers={}, model="gemini-2.5-flash", litellm_params=None) assert headers["Api-Revision"] == "2026-05-20" - def test_api_revision_legacy_schema_when_flag_set(self, config, monkeypatch: pytest.MonkeyPatch): - # Flag on → legacy outputs schema until June 8, 2026 - monkeypatch.setattr(litellm, "use_legacy_interactions_schema", True) - headers = config.validate_environment( - headers={}, model="gemini-2.5-flash", litellm_params=None - ) - assert headers["Api-Revision"] == "2026-05-07" - class TestGetCompleteUrl: def test_url_excludes_api_key(self, config): @@ -158,9 +145,7 @@ class TestTransformRequest: assert request_body["agent"] == "my-custom-slides-agent" assert request_body["environment"] == "remote" assert request_body["stream"] is False - assert request_body["input"] == [ - {"type": "text", "text": "Create a 5-slide presentation about AI trends."} - ] + assert request_body["input"] == [{"type": "text", "text": "Create a 5-slide presentation about AI trends."}] def test_passes_environment_object_to_request_body(self, config): environment_config = { @@ -221,24 +206,15 @@ class TestTransformRequest: class TestStreamingIterator: - def _make_iterator( - self, use_legacy: bool = False - ) -> LiteLLMResponsesInteractionsStreamingIterator: - original = litellm.use_legacy_interactions_schema - litellm.use_legacy_interactions_schema = use_legacy - try: - return LiteLLMResponsesInteractionsStreamingIterator( - model="gpt-5.4", - litellm_custom_stream_wrapper=MagicMock(), - request_input="hi", - optional_params={}, - ) - finally: - litellm.use_legacy_interactions_schema = original + def _make_iterator(self) -> LiteLLMResponsesInteractionsStreamingIterator: + return LiteLLMResponsesInteractionsStreamingIterator( + model="gpt-5.4", + litellm_custom_stream_wrapper=MagicMock(), + request_input="hi", + optional_params={}, + ) - def _make_text_delta( - self, text: str, item_id: str = "item_1" - ) -> OutputTextDeltaEvent: + def _make_text_delta(self, text: str, item_id: str = "item_1") -> OutputTextDeltaEvent: event = MagicMock(spec=OutputTextDeltaEvent) event.delta = text event.item_id = item_id @@ -251,58 +227,29 @@ class TestStreamingIterator: def test_step_delta_includes_type_field(self): """step.delta events must carry delta.type='text' so the UI can display them.""" - it = self._make_iterator(use_legacy=False) + it = self._make_iterator() it.sent_interaction_start = True it.sent_content_start = True - chunk = it._transform_responses_chunk_to_interactions_chunk( - self._make_text_delta("Hello") - ) + chunk = it._transform_responses_chunk_to_interactions_chunk(self._make_text_delta("Hello")) assert chunk is not None assert chunk.event_type == "step.delta" assert chunk.delta == {"type": "text", "text": "Hello"} - def test_content_delta_legacy_schema(self): - """Legacy schema emits content.delta with type and text fields.""" - it = self._make_iterator(use_legacy=True) - it.sent_interaction_start = True - it.sent_content_start = True - - chunk = it._transform_responses_chunk_to_interactions_chunk( - self._make_text_delta("Hello") - ) - - assert chunk is not None - assert chunk.event_type == "content.delta" - assert chunk.delta == {"type": "text", "text": "Hello"} - def test_response_created_emits_interaction_created(self): - it = self._make_iterator(use_legacy=False) + it = self._make_iterator() - chunk = it._transform_responses_chunk_to_interactions_chunk( - self._make_response_created() - ) + chunk = it._transform_responses_chunk_to_interactions_chunk(self._make_response_created()) assert chunk is not None assert chunk.event_type == "interaction.created" assert chunk.id == "resp_123" assert it.sent_interaction_start is True - def test_response_created_emits_interaction_start_legacy(self): - it = self._make_iterator(use_legacy=True) - - chunk = it._transform_responses_chunk_to_interactions_chunk( - self._make_response_created() - ) - - assert chunk is not None - assert chunk.event_type == "interaction.start" - assert chunk.id == "resp_123" - - def test_text_delta_sequence_new_schema(self): + def test_text_delta_sequence(self): """First chunk yields created + step.start + step.delta; later chunks yield step.delta.""" - it = self._make_iterator(use_legacy=False) + it = self._make_iterator() first_events = it._events_for_chunk(self._make_text_delta("Hello")) assert [e.event_type for e in first_events] == [ @@ -322,24 +269,8 @@ class TestStreamingIterator: assert [e.event_type for e in third_events] == ["step.delta"] assert third_events[0].delta == {"type": "text", "text": "!"} - def test_text_delta_sequence_legacy_schema(self): - """Legacy: first chunk yields interaction.start + content.start + content.delta.""" - it = self._make_iterator(use_legacy=True) - - first_events = it._events_for_chunk(self._make_text_delta("Hello")) - assert [e.event_type for e in first_events] == [ - "interaction.start", - "content.start", - "content.delta", - ] - assert first_events[-1].delta == {"type": "text", "text": "Hello"} - - second_events = it._events_for_chunk(self._make_text_delta(" World")) - assert [e.event_type for e in second_events] == ["content.delta"] - assert second_events[0].delta == {"type": "text", "text": " World"} - def test_first_text_delta_without_item_id_uses_fallback_id(self): - it = self._make_iterator(use_legacy=False) + it = self._make_iterator() event = self._make_text_delta("Hi") event.item_id = None @@ -350,11 +281,9 @@ class TestStreamingIterator: def test_first_text_delta_emits_text_via_compat_shim(self): """The legacy single-chunk shim must surface the synthetic events AND the delta.""" - it = self._make_iterator(use_legacy=False) + it = self._make_iterator() - first = it._transform_responses_chunk_to_interactions_chunk( - self._make_text_delta("Hello") - ) + first = it._transform_responses_chunk_to_interactions_chunk(self._make_text_delta("Hello")) assert first is not None assert first.event_type == "interaction.created" @@ -369,7 +298,7 @@ class TestStreamingIterator: def test_response_created_then_text_delta_emits_step_start_and_delta(self): """Realistic flow: response.created arrives first, then text delta.""" - it = self._make_iterator(use_legacy=False) + it = self._make_iterator() first = it._events_for_chunk(self._make_response_created()) assert [e.event_type for e in first] == ["interaction.created"] @@ -380,7 +309,7 @@ class TestStreamingIterator: def test_no_text_token_is_dropped_during_streaming(self): """Concatenated step.delta payloads must equal the upstream text.""" - it = self._make_iterator(use_legacy=False) + it = self._make_iterator() chunks = ["Hello", " ", "world", "!"] emitted_text = "" @@ -401,17 +330,12 @@ class TestStreamingIterator: sync_iter.__iter__ = lambda self: self sync_iter.__next__ = MagicMock(side_effect=[text_event, StopIteration]) - original = litellm.use_legacy_interactions_schema - litellm.use_legacy_interactions_schema = False - try: - it = LiteLLMResponsesInteractionsStreamingIterator( - model="gpt-5.4", - litellm_custom_stream_wrapper=sync_iter, - request_input="hi", - optional_params={}, - ) - finally: - litellm.use_legacy_interactions_schema = original + it = LiteLLMResponsesInteractionsStreamingIterator( + model="gpt-5.4", + litellm_custom_stream_wrapper=sync_iter, + request_input="hi", + optional_params={}, + ) emitted: list = [] try: @@ -450,17 +374,12 @@ class TestStreamingIterator: sync_iter.__iter__ = lambda self: self sync_iter.__next__ = MagicMock(side_effect=[text_event, completed]) - original = litellm.use_legacy_interactions_schema - litellm.use_legacy_interactions_schema = False - try: - it = LiteLLMResponsesInteractionsStreamingIterator( - model="gpt-5.4", - litellm_custom_stream_wrapper=sync_iter, - request_input="hi", - optional_params={}, - ) - finally: - litellm.use_legacy_interactions_schema = original + it = LiteLLMResponsesInteractionsStreamingIterator( + model="gpt-5.4", + litellm_custom_stream_wrapper=sync_iter, + request_input="hi", + optional_params={}, + ) emitted: list = [] try: @@ -506,9 +425,7 @@ class TestInteractionOperationUrls: ), ], ) - def test_url_excludes_key( - self, config, method_name, interaction_id, expected_suffix - ): + def test_url_excludes_key(self, config, method_name, interaction_id, expected_suffix): with patch(_PATCH_GET_API_KEY, return_value="secret-key"): url, params = getattr(config, method_name)( interaction_id=interaction_id, @@ -550,8 +467,7 @@ class TestInteractionOperationUrls: class TestTransformRequestSchemaCoalescing: """Test new-schema request coalescing (Api-Revision: 2026-05-20).""" - def test_response_mime_type_folded_into_response_format(self, config, monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(litellm, "use_legacy_interactions_schema", False) + def test_response_mime_type_folded_into_response_format(self, config): body = config.transform_request( model="gemini/gemini-2.5-flash", agent=None, @@ -571,8 +487,7 @@ class TestTransformRequestSchemaCoalescing: assert rf["mime_type"] == "application/json" assert "schema" in rf - def test_image_config_moved_to_response_format(self, config, monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(litellm, "use_legacy_interactions_schema", False) + def test_image_config_moved_to_response_format(self, config): body = config.transform_request( model="gemini/gemini-2.5-flash", agent=None, @@ -594,9 +509,8 @@ class TestTransformRequestSchemaCoalescing: assert rf["type"] == "image" assert rf["aspect_ratio"] == "1:1" - def test_response_mime_type_skipped_when_response_format_is_list(self, config, monkeypatch: pytest.MonkeyPatch): + def test_response_mime_type_skipped_when_response_format_is_list(self, config): """Lists are already polymorphic; do not wrap them into schema.""" - monkeypatch.setattr(litellm, "use_legacy_interactions_schema", False) rf_list = [ {"type": "text", "mime_type": "application/json"}, {"type": "image", "aspect_ratio": "1:1"}, @@ -619,10 +533,8 @@ class TestTransformRequestSchemaCoalescing: def test_image_config_appended_to_response_format_list_without_mutating_input( self, config, - monkeypatch: pytest.MonkeyPatch, ): """When response_format is already a list, image_config must not mutate optional_params.""" - monkeypatch.setattr(litellm, "use_legacy_interactions_schema", False) text_rf = {"type": "text", "mime_type": "application/json"} optional_params = { "response_format": [text_rf], @@ -659,20 +571,3 @@ class TestTransformRequestSchemaCoalescing: ) assert len(optional_params["response_format"]) == 1 assert body_retry["response_format"] == body["response_format"] - - def test_legacy_schema_passes_fields_unchanged(self, config, monkeypatch: pytest.MonkeyPatch): - monkeypatch.setattr(litellm, "use_legacy_interactions_schema", True) - body = config.transform_request( - model="gemini/gemini-2.5-flash", - agent=None, - input="hello", - optional_params={ - "response_mime_type": "application/json", - "generation_config": {"image_config": {"aspect_ratio": "16:9"}}, - }, - litellm_params=GenericLiteLLMParams(), - headers={}, - ) - - assert body["response_mime_type"] == "application/json" - assert body["generation_config"]["image_config"]["aspect_ratio"] == "16:9" diff --git a/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py b/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py index 13d6cd8a68c..482294e7b92 100644 --- a/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py +++ b/tests/test_litellm/proxy/agent_endpoints/test_endpoints.py @@ -1082,11 +1082,10 @@ class _DbBackedProxyConfig: db_param_value: Final[dict[str, object]] = json.loads(self.stored_litellm_settings_json) if not db_param_value: return config - return ProxyConfig()._update_config_fields( - current_config=config, - param_name="litellm_settings", - db_param_value=db_param_value, - ) + proxy_config: Final = ProxyConfig() + db_values: Final = proxy_config._prepared_db_settings_values("litellm_settings", db_param_value) + proxy_config._apply_litellm_settings_db_values(db_values) + return {"litellm_settings": dict(proxy_config.litellm_settings.resolved())} async def save_config(self, new_config: dict[str, dict[str, object]]) -> None: self.stored_litellm_settings_json = json.dumps(new_config.get("litellm_settings") or {}) diff --git a/tests/test_litellm/proxy/config_resolvers/test_settings_rules.py b/tests/test_litellm/proxy/config_resolvers/test_settings_rules.py new file mode 100644 index 00000000000..ea5ebe6cf12 --- /dev/null +++ b/tests/test_litellm/proxy/config_resolvers/test_settings_rules.py @@ -0,0 +1,186 @@ +from __future__ import annotations + +import itertools +from typing import Final + +import pytest + +from litellm.proxy.config_resolvers.settings_rules import ( + ABSENT, + DUAL_SOURCE_KEYS, + Absent, + JsonValue, + Section, + SettingValue, + is_absent, + resolve, + rule_for, +) +from litellm.proxy.config_resolvers.settings_store import SettingsStore + +_SECTIONS: Final[tuple[Section, ...]] = ( + "general_settings", + "router_settings", + "litellm_settings", + "environment_variables", +) + +_ROUTES: Final[tuple[tuple[Section, str], ...]] = ( + ("general_settings", "max_parallel_requests"), + ("general_settings", "max_file_size_mb"), + ("general_settings", "alerting"), + ("general_settings", "pass_through_endpoints"), + ("general_settings", "forward_client_headers_to_llm_api"), + ("router_settings", "fallbacks"), + ("litellm_settings", "drop_params"), + ("general_settings", "an_unregistered_key"), +) + +_CONFIG_VALUES: Final[tuple[SettingValue, ...]] = ( + ABSENT, + None, + False, + 0, + "", + [], + {}, + "config-value", + ["config-value"], + {"config": "value"}, + [{"path": "/shared", "target": "config"}], +) + +_DB_VALUES: Final[tuple[SettingValue, ...]] = ( + ABSENT, + None, + False, + 0, + "", + [], + {}, + "db-value", + ["db-value"], + {"db": "value"}, + [{"path": "/shared", "target": "db"}], +) + +_MATRIX: Final = tuple( + (section, key, config_value, db_value) + for (section, key), config_value, db_value in itertools.product(_ROUTES, _CONFIG_VALUES, _DB_VALUES) +) + +_PREVIOUSLY_DB_WINS: Final[tuple[str, ...]] = ( + "max_parallel_requests", + "global_max_parallel_requests", + "alerting_args", + "ui_access_mode", + "disable_auto_add_proxy_admin_to_teams", + "store_model_in_db", + "maximum_spend_logs_retention_period", + "maximum_autorouter_session_retention_period", + "maximum_health_check_retention_period", + "maximum_spend_logs_cleanup_batch_size", + "maximum_spend_logs_cleanup_max_batches", + "maximum_spend_logs_cleanup_run_budget", + "maximum_spend_logs_cleanup_batch_timeout", + "user_url_validation", + "user_url_allowed_hosts", + "provider_url_destination_allowed_hosts", + "alerting", + "pass_through_endpoints", +) + + +def _store_for(section: Section, key: str, config_value: SettingValue, db_value: SettingValue) -> SettingsStore: + store: Final = SettingsStore(section) + store.load_yaml({} if is_absent(config_value) else {key: config_value}) + if not is_absent(db_value): + store.apply_db_row(rule_for(section, key).db_row, {key: db_value}) + return store + + +@pytest.mark.parametrize(("section", "key", "config_value", "db_value"), _MATRIX) +def test_the_store_resolves_every_config_and_stored_value_combination( + section: Section, key: str, config_value: SettingValue, db_value: SettingValue +) -> None: + store: Final = _store_for(section, key, config_value, db_value) + + if not is_absent(config_value): + assert store[key] == config_value + assert store.source(key) == "config" + elif is_absent(db_value) or db_value is None: + assert key not in store + assert store.source(key) == "unset" + else: + assert store[key] == db_value + assert store.source(key) == "db" + + +@pytest.mark.parametrize(("section", "key", "config_value", "db_value"), _MATRIX) +def test_the_store_and_the_resolver_never_disagree( + section: Section, key: str, config_value: SettingValue, db_value: SettingValue +) -> None: + resolved: Final = resolve(config_value, db_value) + store: Final = _store_for(section, key, config_value, db_value) + + assert store.source(key) == resolved.source + if isinstance(resolved.value, Absent): + assert key not in store + else: + assert store[key] == resolved.value + + +@pytest.mark.parametrize(("section", "key"), _ROUTES) +def test_a_stored_row_the_key_does_not_belong_to_never_reaches_it(section: Section, key: str) -> None: + other_row: Final = "ui_settings" if rule_for(section, key).db_row != "ui_settings" else "general_settings" + store: Final = SettingsStore(section) + store.load_yaml({}) + store.apply_db_row(other_row, {key: "from-the-wrong-row"}) + + assert key not in store + assert store.source(key) == "unset" + + +@pytest.mark.parametrize("key", _PREVIOUSLY_DB_WINS) +def test_keys_the_database_used_to_win_now_resolve_to_the_config_value(key: str) -> None: + store: Final = _store_for("general_settings", key, "from-config", "from-db") + + assert store[key] == "from-config" + assert store.source(key) == "config" + + +@pytest.mark.parametrize("key", _PREVIOUSLY_DB_WINS) +def test_a_falsy_stored_value_cannot_erase_a_config_value(key: str) -> None: + falsy: Final[tuple[JsonValue, ...]] = (None, False, 0, "", [], {}) + + stores: Final = tuple(_store_for("general_settings", key, "from-config", value) for value in falsy) + + assert {store[key] for store in stores} == {"from-config"} + assert {store.source(key) for store in stores} == {"config"} + + +@pytest.mark.parametrize( + ("key", "expected_row"), + ( + ("forward_client_headers_to_llm_api", "ui_settings"), + ("team_admin_editable_team_fields", "ui_settings"), + ("disable_key_generate_for_org_admin", "ui_settings"), + ("max_parallel_requests", "general_settings"), + ("an_unregistered_key", "general_settings"), + ), +) +def test_a_key_reads_from_the_row_that_carries_it(key: str, expected_row: str) -> None: + assert rule_for("general_settings", key).db_row == expected_row + + +def test_every_registered_rule_routes_to_a_known_row() -> None: + rows: Final = {rule.db_row for rule in DUAL_SOURCE_KEYS.values()} + + assert rows <= {*_SECTIONS, "ui_settings"} + + +def test_a_config_value_of_none_is_still_config_owned() -> None: + resolved: Final = resolve(None, "from-db") + + assert resolved.value is None + assert resolved.source == "config" diff --git a/tests/test_litellm/proxy/config_resolvers/test_settings_store.py b/tests/test_litellm/proxy/config_resolvers/test_settings_store.py new file mode 100644 index 00000000000..aa410deac43 --- /dev/null +++ b/tests/test_litellm/proxy/config_resolvers/test_settings_store.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +from typing import Final + +import pytest + +from litellm.proxy.config_resolvers.settings_rules import JsonValue +from litellm.proxy.config_resolvers.settings_store import SettingsStore + + +def test_settings_store_matches_plain_dict_mapping_operations() -> None: + store: Final = SettingsStore("general_settings") + + store["none"] = None + store["false"] = False + store["zero"] = 0 + store["empty_list"] = [] + store["empty_string"] = "" + store.update({"updated": "value"}) + defaulted: Final = store.setdefault("defaulted", "default") + existing: Final = store.setdefault("updated", "other") + popped: Final = store.pop("updated") + + assert defaulted == "default" + assert existing == "value" + assert popped == "value" + assert store.get("missing") is None + assert store["none"] is None + assert "false" in store + assert tuple(store) == ("none", "false", "zero", "empty_list", "empty_string", "defaulted") + assert len(store) == 6 + assert dict(store) == { + "none": None, + "false": False, + "zero": 0, + "empty_list": [], + "empty_string": "", + "defaulted": "default", + } + + +@pytest.mark.parametrize("operation", ("set", "update", "setdefault", "pop", "delete")) +@pytest.mark.parametrize("initial_value", (None, False, 0, [], "")) +def test_settings_store_mapping_operations_match_a_plain_dict(operation: str, initial_value: JsonValue) -> None: + expected: dict[str, JsonValue] = {"value": initial_value} + store: Final = SettingsStore("general_settings") + store["value"] = initial_value + + match operation: + case "set": + expected["value"] = "replacement" + store["value"] = "replacement" + case "update": + expected.update({"value": "replacement", "other": initial_value}) + store.update({"value": "replacement", "other": initial_value}) + case "setdefault": + assert store.setdefault("value", "replacement") == expected.setdefault("value", "replacement") + assert store.setdefault("other", initial_value) == expected.setdefault("other", initial_value) + case "pop": + assert store.pop("value") == expected.pop("value") + case "delete": + del expected["value"] + del store["value"] + case _: + raise AssertionError(f"unexpected operation: {operation}") + + assert dict(store) == expected + assert tuple(store) == tuple(expected) + assert len(store) == len(expected) + assert ("value" in store) is ("value" in expected) + + +def test_settings_store_keeps_unaffected_runtime_values_on_a_db_row_refresh() -> None: + store: Final = SettingsStore("general_settings") + store.load_yaml({"template": "os.environ/SETTING"}) + store.apply_runtime_values({"template": "resolved", "changed": "resolved-runtime"}) + + store.apply_db_row("general_settings", {"changed": "database"}) + + assert store["template"] == "resolved" + assert store["changed"] == "database" + assert store.source("changed") == "db" + + +def test_settings_store_keeps_a_config_owned_key_when_a_db_row_disagrees() -> None: + store: Final = SettingsStore("general_settings") + store.load_yaml({"changed": "config"}) + store.apply_runtime_values({"changed": "resolved-config"}) + + store.apply_db_row("general_settings", {"changed": "database"}) + + assert store["changed"] == "config" + assert store.source("changed") == "config" + + +def test_settings_store_removes_only_runtime_values_affected_by_a_cleared_db_row() -> None: + store: Final = SettingsStore("general_settings") + store.load_yaml({"template": "os.environ/SETTING"}) + store.apply_db_row("ui_settings", {"allow_public_health_readiness_details": True}) + store.apply_runtime_values({"template": "resolved", "allow_public_health_readiness_details": True}) + + store.apply_db_row("ui_settings", {}) + + assert store["template"] == "resolved" + assert "allow_public_health_readiness_details" not in store + + +def test_settings_store_preserves_falsy_config_values_and_provenance() -> None: + store: Final = SettingsStore("general_settings") + yaml_values: Final = {"none": None, "false": False, "zero": 0, "empty_list": [], "empty_string": ""} + + store.load_yaml(yaml_values) + + assert dict(store) == yaml_values + assert tuple(store.source(key) for key in yaml_values) == ("config",) * len(yaml_values) + + +@pytest.mark.parametrize( + ("yaml_value", "db_value", "expected_value", "expected_source"), + ( + ("from-config", "from-db", "from-config", "config"), + ("from-config", None, "from-config", "config"), + (None, "from-db", None, "config"), + (None, None, None, "config"), + ), +) +def test_settings_store_resolves_a_db_row_with_provenance( + yaml_value: object, + db_value: object, + expected_value: object, + expected_source: str, +) -> None: + store: Final = SettingsStore("general_settings") + store.load_yaml({"ordinary": yaml_value}) + store.apply_db_row("general_settings", {"ordinary": db_value}) + + assert store["ordinary"] == expected_value + assert store.source("ordinary") == expected_source + + +def test_settings_store_gives_every_config_declared_key_to_the_config_file() -> None: + store: Final = SettingsStore("general_settings") + store.load_yaml({"max_file_size_mb": 7, "max_parallel_requests": 3}) + store.apply_db_row("general_settings", {"max_file_size_mb": 9, "max_parallel_requests": 11}) + + assert dict(store) == {"max_file_size_mb": 7, "max_parallel_requests": 3} + assert store.source("max_file_size_mb") == "config" + assert store.source("max_parallel_requests") == "config" + + +def test_settings_store_gives_a_key_the_config_file_omits_to_the_database() -> None: + store: Final = SettingsStore("general_settings") + store.load_yaml({"max_file_size_mb": 7}) + store.apply_db_row("general_settings", {"max_file_size_mb": 9, "max_parallel_requests": 11}) + + assert dict(store) == {"max_file_size_mb": 7, "max_parallel_requests": 11} + assert store.source("max_parallel_requests") == "db" + + +def test_settings_store_refuses_a_runtime_write_to_a_config_owned_key() -> None: + store: Final = SettingsStore("general_settings") + store.load_yaml({"max_parallel_requests": 3}) + + store["max_parallel_requests"] = 11 + del store["max_parallel_requests"] + + assert store["max_parallel_requests"] == 3 + assert store.source("max_parallel_requests") == "config" + + +def test_settings_store_reports_the_config_owned_keys_a_write_would_change() -> None: + store: Final = SettingsStore("general_settings") + store.load_yaml({"max_parallel_requests": 3, "ui_access_mode": "admin_only"}) + + rejected: Final = store.rejected_writes( + {"max_parallel_requests": 11, "ui_access_mode": "admin_only", "global_max_parallel_requests": 5} + ) + + assert rejected == ("max_parallel_requests",) + + +def test_settings_store_resolved_view_is_read_only() -> None: + store: Final = SettingsStore("general_settings") + store.load_yaml({"configured": "value"}) + resolved: Final = store.resolved() + + with pytest.raises(TypeError): + resolved["configured"] = "changed" + + assert store["configured"] == "value" + + +def test_settings_store_omits_a_null_database_overlay_value() -> None: + store: Final = SettingsStore("router_settings") + store.apply_db_row("router_settings", {"fallbacks": None}) + + assert "fallbacks" not in store + assert dict(store) == {} + assert store.source("fallbacks") == "unset" + + +def test_settings_store_keeps_an_empty_database_list_without_a_config_value() -> None: + store: Final = SettingsStore("router_settings") + store.apply_db_row("router_settings", {"fallbacks": []}) + + assert store["fallbacks"] == [] + assert store.source("fallbacks") == "db" + + +@pytest.mark.asyncio +async def test_load_config_returns_and_binds_the_general_settings_store(tmp_path, monkeypatch) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.proxy_server import ProxyConfig + + config_path = tmp_path / "config.yaml" + config_path.write_text("model_list: []\ngeneral_settings:\n max_file_size_mb: 5\n") + monkeypatch.setattr(proxy_server, "prisma_client", None) + monkeypatch.setattr(proxy_server, "store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + + proxy_config: Final = ProxyConfig() + _router, _models, returned_store = await proxy_config.load_config(router=None, config_file_path=str(config_path)) + + config_state: Final = proxy_config.get_config_state() + + assert returned_store is proxy_config.settings + assert proxy_server.general_settings is proxy_config.settings + assert isinstance(config_state["general_settings"], dict) + assert config_state["general_settings"]["max_file_size_mb"] == 5 + + +def test_settings_store_starts_with_an_unset_source() -> None: + store: Final = SettingsStore("general_settings") + + assert store.source("unknown") == "unset" diff --git a/tests/test_litellm/proxy/management_endpoints/jwt_key_mapping_doubles.py b/tests/test_litellm/proxy/management_endpoints/jwt_key_mapping_doubles.py new file mode 100644 index 00000000000..8722e139ad1 --- /dev/null +++ b/tests/test_litellm/proxy/management_endpoints/jwt_key_mapping_doubles.py @@ -0,0 +1,27 @@ +"""LiteLLM_JWTKeyMapping test doubles for the bulk key deletion paths.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + + +@dataclass(frozen=True, slots=True) +class JWTMappingRow: + token: str + jwt_claim_name: str + jwt_claim_value: str + jwt_issuer: str | None = None + + +class CascadingJWTMappingTable: + """Mapping rows that LiteLLM_JWTKeyMapping_token_fkey drops when their key row is deleted.""" + + def __init__(self, rows: Sequence[JWTMappingRow]) -> None: + self.rows: tuple[JWTMappingRow, ...] = tuple(rows) + + async def find_many(self, where: Mapping[str, Mapping[str, Sequence[str]]]) -> list[JWTMappingRow]: + return [row for row in self.rows if row.token in where["token"]["in"]] + + def cascade(self, deleted_tokens: Sequence[str]) -> None: + self.rows = tuple(row for row in self.rows if row.token not in deleted_tokens) diff --git a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py index c73d29e78b2..8c3e88b864a 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py +++ b/tests/test_litellm/proxy/management_endpoints/test_cost_tracking_settings.py @@ -733,11 +733,11 @@ class TestBlockRequestsForModelsWithoutPricing: from litellm.proxy.proxy_server import ProxyConfig with patch.object(litellm, "block_requests_for_models_without_pricing", False): - ProxyConfig()._update_config_fields( - current_config={}, - param_name="litellm_settings", - db_param_value={"block_requests_for_models_without_pricing": True}, + proxy_config = ProxyConfig() + db_values = proxy_config._prepared_db_settings_values( + "litellm_settings", {"block_requests_for_models_without_pricing": True} ) + proxy_config._apply_litellm_settings_db_values(db_values) assert litellm.block_requests_for_models_without_pricing is True diff --git a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py index 0d8b19345f1..3f2ba365a04 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_internal_user_endpoints.py @@ -4,11 +4,10 @@ from types import SimpleNamespace from typing import Final import pytest -from fastapi.testclient import TestClient from fastapi import HTTPException +from fastapi.testclient import TestClient from pytest_mock import MockerFixture - from litellm.proxy._types import ( LiteLLM_UserTableFiltered, LitellmUserRoles, @@ -27,6 +26,10 @@ from litellm.proxy.management_endpoints.internal_user_endpoints import ( ui_view_users, ) from litellm.proxy.proxy_server import app +from tests.test_litellm.proxy.management_endpoints.jwt_key_mapping_doubles import ( + CascadingJWTMappingTable, + JWTMappingRow, +) client = TestClient(app) @@ -2627,6 +2630,9 @@ async def test_delete_user_cleans_up_created_by_invitation_links(mocker): ) # Mock all delete_many calls + mock_prisma_client.db.litellm_verificationtoken.find_many = mocker.AsyncMock( + return_value=[] + ) mock_prisma_client.db.litellm_verificationtoken.delete_many = mocker.AsyncMock( return_value=0 ) @@ -2676,6 +2682,84 @@ async def test_delete_user_cleans_up_created_by_invitation_links(mocker): assert condition[field] == {"in": ["admin-creator"]} +@pytest.mark.asyncio +async def test_delete_user_evicts_jwt_key_mapping_cache_of_its_keys(mocker): + """/user/delete bulk-deletes the user's keys without going through /key/delete, so the + jwt_key_mapping cache entries pointing at those keys must be evicted here too. A surviving + entry keeps resolving the deleted token hash until the mapping cache TTL expires: the deleted + identity is either still served through the stale key cache or 401s on every JWT call, and it is + never re-registered (LIT-5387). + + The FK cascade drops the mapping rows with the key rows, so the cache keys have to be read + before the delete: reading them afterwards finds nothing to evict. + """ + from litellm.proxy._types import DeleteUserRequest, UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.management_endpoints.internal_user_endpoints import delete_user + + global_cache_key: Final = jwt_key_mapping_cache_key("sub", "jwt-user", None) + issuer_cache_key: Final = jwt_key_mapping_cache_key("sub", "jwt-user", "https://issuer.example") + unrelated_cache_key: Final = jwt_key_mapping_cache_key("sub", "other-user", None) + jwt_table: Final = CascadingJWTMappingTable( + [ + JWTMappingRow("hashed-jwt-key", "sub", "jwt-user"), + JWTMappingRow("hashed-issuer-key", "sub", "jwt-user", "https://issuer.example"), + JWTMappingRow("hashed-unrelated-key", "sub", "other-user"), + ] + ) + cache: Final = UserApiKeyCache() + for cache_key, hashed_token in ( + (global_cache_key, "hashed-jwt-key"), + (issuer_cache_key, "hashed-issuer-key"), + (unrelated_cache_key, "hashed-unrelated-key"), + ): + cache.set_cache(key=cache_key, value=hashed_token) + cache.set_cache(key=hashed_token, value=UserAPIKeyAuth(token=hashed_token)) + + user_row: Final = mocker.MagicMock() + user_row.user_id = "jwt-user" + user_row.user_email = "jwt-user@example.com" + user_row.teams = [] + user_row.model_dump_json.return_value = "{}" + user_row.model_dump.return_value = {"user_id": "jwt-user", "user_email": "jwt-user@example.com", "teams": []} + + mock_prisma_client: Final = mocker.MagicMock() + mock_prisma_client.db.litellm_usertable.find_unique = mocker.AsyncMock(return_value=user_row) + mock_prisma_client.db.litellm_teamtable.find_many = mocker.AsyncMock(return_value=[]) + mock_prisma_client.db.litellm_jwtkeymapping = jwt_table + mock_prisma_client.db.litellm_verificationtoken.find_many = mocker.AsyncMock( + return_value=[SimpleNamespace(token="hashed-jwt-key"), SimpleNamespace(token="hashed-issuer-key")] + ) + + async def cascading_delete_many(where): + jwt_table.cascade(("hashed-jwt-key", "hashed-issuer-key")) + return 2 + + mock_prisma_client.db.litellm_verificationtoken.delete_many = mocker.AsyncMock(side_effect=cascading_delete_many) + mock_prisma_client.db.litellm_invitationlink.delete_many = mocker.AsyncMock(return_value=0) + mock_prisma_client.db.litellm_organizationmembership.delete_many = mocker.AsyncMock(return_value=0) + mock_prisma_client.db.litellm_teammembership.delete_many = mocker.AsyncMock(return_value=0) + mock_prisma_client.db.litellm_usertable.delete_many = mocker.AsyncMock(return_value=1) + + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) # test-quality-ok: substitute the database dependency + mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache) # test-quality-ok: exercise a real isolated cache + mocker.patch("litellm.proxy.proxy_server.proxy_logging_obj", None) # test-quality-ok: delete_user reads it off proxy_server at call time + + await delete_user( + data=DeleteUserRequest(user_ids=["jwt-user"]), + user_api_key_dict=UserAPIKeyAuth(user_id="proxy-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert cache.get_cache(key=global_cache_key) is None + assert cache.get_cache(key=issuer_cache_key) is None + assert cache.get_cache(key="hashed-jwt-key") is None + assert cache.get_cache(key="hashed-issuer-key") is None + assert cache.get_cache(key=unrelated_cache_key) == "hashed-unrelated-key" + assert cache.get_cache(key="hashed-unrelated-key") is not None + assert [row.token for row in jwt_table.rows] == ["hashed-unrelated-key"] + + @pytest.mark.asyncio async def test_delete_user_rejects_org_admin_deleting_outside_scope(mocker): """Regression: an org admin of org-A must not be able to delete a user diff --git a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py index cc0a7631b59..3b26da8e7ac 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_key_management_endpoints.py @@ -5244,7 +5244,10 @@ async def test_delete_verification_tokens_evicts_jwt_key_mapping_cache(monkeypat virtual_key_mapping_cache_ttl expires, instead of auto-registering again. """ jwt_table = _CascadingJWTMappingTable( - [_JWTMappingRow("hashed-token-1", "email", "user@example.com")] + [ + _JWTMappingRow("hashed-token-1", "email", "user@example.com"), + _JWTMappingRow("hashed-token-1", "email", "user@example.com", "https://issuer.example"), + ] ) key1 = LiteLLM_VerificationToken( @@ -5302,7 +5305,10 @@ async def test_delete_verification_tokens_evicts_jwt_key_mapping_cache(monkeypat ), ) - assert recording_evict.cache_keys == (jwt_key_mapping_cache_key("email", "user@example.com", None),) + assert recording_evict.cache_keys == ( + jwt_key_mapping_cache_key("email", "user@example.com", None), + jwt_key_mapping_cache_key("email", "user@example.com", "https://issuer.example"), + ) @pytest.mark.asyncio diff --git a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py index 47ee5dc1dd2..47acc091672 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_organization_endpoints.py @@ -1,7 +1,6 @@ import asyncio import json -from litellm._uuid import uuid -from types import MappingProxyType +from types import MappingProxyType, SimpleNamespace from typing import Final, Mapping, Optional, cast from unittest.mock import AsyncMock, MagicMock, patch @@ -9,6 +8,11 @@ import pytest from fastapi import HTTPException from fastapi.testclient import TestClient +from litellm._uuid import uuid +from tests.test_litellm.proxy.management_endpoints.jwt_key_mapping_doubles import ( + CascadingJWTMappingTable, + JWTMappingRow, +) @pytest.mark.asyncio @@ -499,9 +503,10 @@ async def test_organization_info_includes_user_email(monkeypatch): """ Test that GET /organization/info returns user_email in members list. """ - from litellm.proxy._types import LiteLLM_OrganizationMembershipTable from datetime import datetime + from litellm.proxy._types import LiteLLM_OrganizationMembershipTable + # Simulate a membership row with a nested user object that has user_email raw_membership = { "user_id": "user_abc", @@ -573,6 +578,10 @@ async def test_organization_member_add_rejects_unauthorized_caller(patched_org_p # ``organization_member_add`` catches HTTPException in its # catch-all and re-wraps as ProxyException with the original status # code preserved. + from unittest.mock import Mock + + from fastapi import Request + from litellm.proxy._types import ( OrganizationMemberAddRequest, OrgMember, @@ -581,9 +590,6 @@ async def test_organization_member_add_rejects_unauthorized_caller(patched_org_p from litellm.proxy.management_endpoints.organization_endpoints import ( organization_member_add, ) - from unittest.mock import Mock - - from fastapi import Request data = OrganizationMemberAddRequest( organization_id="org-victim", @@ -1438,3 +1444,61 @@ def test_organization_routes_reach_their_handler_with_enterprise_license(monkeyp assert any( message in response.text for message in (CommonProxyErrors.db_not_connected_error.value, "No db connected") ) + + +@pytest.mark.asyncio +async def test_delete_organization_evicts_the_cache_of_the_keys_it_deletes(monkeypatch): + """/organization/delete bulk-deletes the org's keys without going through /key/delete, so the + key objects and the jwt_key_mapping entries (issuer-scoped ones included) pointing at them + must be evicted here, or a deleted key keeps authenticating and a JWT identity keeps resolving + a token hash that no longer exists until the TTLs expire. The FK cascade drops the mapping + rows with the key rows, so the cache keys have to be read before the delete (LIT-5387).""" + from litellm.proxy._types import DeleteOrganizationRequest, LitellmUserRoles, UserAPIKeyAuth + from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.management_endpoints.organization_endpoints import delete_organization + + doomed_cache_keys: Final = ( + "hashed-org-key", + jwt_key_mapping_cache_key("sub", "svc-account", None), + jwt_key_mapping_cache_key("sub", "svc-account", "https://issuer.example"), + ) + kept_cache_keys: Final = ("hashed-other-key", jwt_key_mapping_cache_key("sub", "other-account", None)) + kept_row: Final = JWTMappingRow("hashed-other-key", "sub", "other-account") + jwt_table: Final = CascadingJWTMappingTable( + [ + JWTMappingRow("hashed-org-key", "sub", "svc-account"), + JWTMappingRow("hashed-org-key", "sub", "svc-account", "https://issuer.example"), + kept_row, + ] + ) + cache: Final = UserApiKeyCache() + for cache_key in (*doomed_cache_keys, *kept_cache_keys): + cache.set_cache(key=cache_key, value={"retained": True}) + + prisma_client: Final = AsyncMock() + prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[SimpleNamespace(token="hashed-org-key")] + ) + + async def cascading_delete_many(where): + jwt_table.cascade(("hashed-org-key",)) + return 1 + + prisma_client.db.litellm_verificationtoken.delete_many = AsyncMock(side_effect=cascading_delete_many) + prisma_client.db.litellm_jwtkeymapping = jwt_table + prisma_client.db.litellm_organizationtable.delete = AsyncMock(return_value=MagicMock()) + + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True, raising=False) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + monkeypatch.setattr("litellm.proxy.proxy_server.proxy_logging_obj", None) + + await delete_organization( + data=DeleteOrganizationRequest(organization_ids=["org-doomed"]), + user_api_key_dict=UserAPIKeyAuth(api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + assert all(cache.get_cache(key=cache_key) is None for cache_key in doomed_cache_keys) + assert all(cache.get_cache(key=cache_key) == {"retained": True} for cache_key in kept_cache_keys) + assert jwt_table.rows == (kept_row,) diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py b/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py index 265437f97e9..17cb30dd07d 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_default_params.py @@ -24,13 +24,7 @@ from litellm.proxy.management_endpoints.team_endpoints import ( from litellm.proxy.proxy_server import ProxyConfig -# --------------------------------------------------------------------------- -# _update_config_fields: default_team_params loaded from DB on startup -# --------------------------------------------------------------------------- - - -class TestConfigFieldsDefaultTeamParams: - """Tests that _update_config_fields applies default_team_params from DB.""" +class TestDefaultTeamParamsFromSettingsStore: def _make_proxy_config(self) -> ProxyConfig: return ProxyConfig() @@ -50,11 +44,8 @@ class TestConfigFieldsDefaultTeamParams: } } - pc._update_config_fields( - current_config={}, - param_name="litellm_settings", - db_param_value=db_settings, - ) + db_values = pc._prepared_db_settings_values("litellm_settings", db_settings) + pc._apply_litellm_settings_db_values(db_values) assert litellm.default_team_params == db_settings["default_team_params"] @@ -68,11 +59,9 @@ class TestConfigFieldsDefaultTeamParams: } } - result = pc._update_config_fields( - current_config=config, - param_name="litellm_settings", - db_param_value=db_settings, - ) + pc.litellm_settings.load_yaml(config["litellm_settings"]) + pc.litellm_settings.apply_db_row("litellm_settings", db_settings) + result = {"litellm_settings": dict(pc.litellm_settings.resolved())} assert result["litellm_settings"]["default_team_params"] == {"max_budget": 100.0} # Existing keys preserved @@ -83,16 +72,14 @@ class TestConfigFieldsDefaultTeamParams: monkeypatch.setattr(litellm, "default_team_params", None) pc = self._make_proxy_config() - pc._update_config_fields( - current_config={}, - param_name="litellm_settings", - db_param_value={"cache": True}, - ) + db_values = pc._prepared_db_settings_values("litellm_settings", {"cache": True}) + pc._apply_litellm_settings_db_values(db_values) assert litellm.default_team_params is None - def test_default_team_params_overrides_yaml_value(self, monkeypatch): - """DB value for default_team_params overrides YAML value via deep merge.""" + def test_default_team_params_keeps_the_yaml_value(self, monkeypatch): + """``default_team_params`` is config-owned once the file declares it, so a stored + value no longer merges into or replaces any part of it.""" monkeypatch.setattr(litellm, "default_team_params", None) pc = self._make_proxy_config() @@ -111,22 +98,29 @@ class TestConfigFieldsDefaultTeamParams: } } - result = pc._update_config_fields( - current_config=config, - param_name="litellm_settings", - db_param_value=db_settings, - ) + pc.litellm_settings.load_yaml(config["litellm_settings"]) + db_values = pc._prepared_db_settings_values("litellm_settings", db_settings) + pc._apply_litellm_settings_db_values(db_values) - merged = result["litellm_settings"]["default_team_params"] - # DB value wins for max_budget - assert merged["max_budget"] == 200.0 - # DB adds rpm_limit - assert merged["rpm_limit"] == 500 - # YAML tpm_limit preserved (not in DB) - assert merged["tpm_limit"] == 100 + resolved = pc.litellm_settings["default_team_params"] + assert resolved == {"max_budget": 50.0, "tpm_limit": 100} + assert pc.litellm_settings.source("default_team_params") == "config" + assert litellm.default_team_params == resolved - # setattr should have applied the DB value - assert litellm.default_team_params == db_settings["default_team_params"] + def test_default_team_params_comes_from_the_database_when_the_yaml_omits_it(self, monkeypatch): + monkeypatch.setattr(litellm, "default_team_params", None) + + pc = self._make_proxy_config() + db_settings = {"default_team_params": {"max_budget": 200.0, "rpm_limit": 500}} + + pc.litellm_settings.load_yaml({}) + db_values = pc._prepared_db_settings_values("litellm_settings", db_settings) + pc._apply_litellm_settings_db_values(db_values) + + resolved = pc.litellm_settings["default_team_params"] + assert resolved == {"max_budget": 200.0, "rpm_limit": 500} + assert pc.litellm_settings.source("default_team_params") == "db" + assert litellm.default_team_params == resolved # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py index a89bc9a8a3e..ec7c989e342 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_endpoints.py @@ -12,8 +12,6 @@ from fastapi.testclient import TestClient from pydantic import ValidationError from litellm._uuid import uuid - -from litellm.proxy._types import UserAPIKeyAuth # Import UserAPIKeyAuth from litellm.proxy._types import ( LiteLLM_BudgetTable, LiteLLM_BudgetTableFull, @@ -33,14 +31,12 @@ from litellm.proxy._types import ( TeamMemberAddRequest, TeamMemberUpdateRequest, UpdateTeamRequest, + UserAPIKeyAuth, # Import UserAPIKeyAuth ) from litellm.proxy.management_endpoints.team_endpoints import ( - user_api_key_auth, # Assuming this dependency is needed -) -from litellm.proxy.management_endpoints.team_endpoints import ( + _STRIP_DELETED_TEAM_FROM_USERS_SQL, GetTeamMemberPermissionsResponse, UpdateTeamMemberPermissionsRequest, - _STRIP_DELETED_TEAM_FROM_USERS_SQL, _persist_deleted_team_records, _save_deleted_team_records, _transform_teams_to_deleted_records, @@ -56,6 +52,7 @@ from litellm.proxy.management_endpoints.team_endpoints import ( team_member_delete, team_member_update, update_team, + user_api_key_auth, # Assuming this dependency is needed validate_team_org_change, ) from litellm.proxy.management_helpers.access_group_team_sync import ( @@ -71,6 +68,10 @@ from litellm.types.proxy.management_endpoints.team_endpoints import ( BulkTeamMemberAddResponse, TeamMemberAddResult, ) +from tests.test_litellm.proxy.management_endpoints.jwt_key_mapping_doubles import ( + CascadingJWTMappingTable, + JWTMappingRow, +) # Setup TestClient client = TestClient(app) @@ -2788,7 +2789,7 @@ async def test_upsert_team_member_budget_table_existing_budget(): """ from unittest.mock import AsyncMock, MagicMock, patch - from litellm.proxy._types import LitellmUserRoles, LiteLLM_TeamTable, UserAPIKeyAuth + from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.management_endpoints.team_endpoints import ( TeamMemberBudgetHandler, ) @@ -2849,7 +2850,7 @@ async def test_upsert_team_member_budget_table_no_existing_budget(): """ from unittest.mock import AsyncMock, MagicMock, patch - from litellm.proxy._types import LitellmUserRoles, LiteLLM_TeamTable, UserAPIKeyAuth + from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.management_endpoints.team_endpoints import ( TeamMemberBudgetHandler, ) @@ -6092,9 +6093,9 @@ async def test_new_team_standalone_validates_against_user_models(monkeypatch): - Team is created WITHOUT organization_id and models=['gpt-4'] - Expected: Should fail with "Model not in allowed user models" """ - import litellm from fastapi import Request + import litellm from litellm.proxy._types import NewTeamRequest, ProxyException, UserAPIKeyAuth from litellm.proxy.management_endpoints.team_endpoints import new_team @@ -9180,6 +9181,154 @@ async def test_team_member_delete_persists_deleted_keys(monkeypatch): assert cache.get_cache(key="unrelated-key") == {"retained": True} +def _seed_jwt_mapping_cache(cache, mapping_rows): + from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key + + cache_keys = tuple( + jwt_key_mapping_cache_key(row.jwt_claim_name, row.jwt_claim_value, row.jwt_issuer) for row in mapping_rows + ) + for cache_key, row in zip(cache_keys, mapping_rows): + cache.set_cache(key=cache_key, value=row.token) + return cache_keys + + +@pytest.mark.asyncio +async def test_team_member_delete_evicts_jwt_key_mapping_cache_of_the_keys_it_deletes(monkeypatch): + """The member's team keys are deleted in bulk here, not through /key/delete, so the + jwt_key_mapping cache entries pointing at them must be evicted here too, or every JWT call + from that identity resolves the deleted token hash and 401s until the mapping TTL expires. + The FK cascade drops the mapping rows with the key rows, so the cache keys have to be read + before the delete (LIT-5387).""" + from litellm.proxy._types import TeamMemberDeleteRequest + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + from litellm.proxy.management_endpoints.key_management_endpoints import LiteLLM_VerificationToken + + doomed_rows: Final = ( + JWTMappingRow("hashed-token-1", "sub", "user-123"), + JWTMappingRow("hashed-token-1", "sub", "user-123", "https://issuer.example"), + ) + kept_row: Final = JWTMappingRow("hashed-other-key", "sub", "user-999") + jwt_table: Final = CascadingJWTMappingTable([*doomed_rows, kept_row]) + + team = LiteLLM_TeamTable( + team_id="team-1", + team_alias="test-team", + members_with_roles=[Member(user_id="user-123", role="admin")], + metadata={}, + model_max_budget={}, + model_spend={}, + ) + key1 = LiteLLM_VerificationToken(token="hashed-token-1", user_id="user-123", team_id="team-1") + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=team) + mock_prisma_client.db.litellm_usertable.find_many = AsyncMock( + return_value=[MagicMock(user_id="user-123", teams=["team-1"])] + ) + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock(return_value=[key1]) + + async def cascading_delete_many(where): + jwt_table.cascade(("hashed-token-1",)) + + mock_prisma_client.db.litellm_verificationtoken.delete_many = AsyncMock(side_effect=cascading_delete_many) + mock_prisma_client.db.litellm_jwtkeymapping = jwt_table + _wire_member_delete_tx(mock_prisma_client) + + cache: Final = UserApiKeyCache() + doomed_cache_keys: Final = _seed_jwt_mapping_cache(cache, doomed_rows) + (kept_cache_key,) = _seed_jwt_mapping_cache(cache, (kept_row,)) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + monkeypatch.setattr("litellm.proxy.management_endpoints.team_endpoints._is_user_team_admin", lambda **kwargs: True) + + await team_member_delete( + data=TeamMemberDeleteRequest(team_id="team-1", user_id="user-123"), + user_api_key_dict=UserAPIKeyAuth( + user_id="admin-user", api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN.value + ), + ) + + assert all(cache.get_cache(key=cache_key) is None for cache_key in doomed_cache_keys) + assert cache.get_cache(key=kept_cache_key) == "hashed-other-key" + assert jwt_table.rows == (kept_row,) + + +@pytest.mark.asyncio +async def test_delete_team_evicts_jwt_key_mapping_cache_of_the_keys_it_deletes( + monkeypatch, + disable_audit_logging_for_mocked_team, +): + """Same contract as /team/member_delete for the bulk key delete in /team/delete: the + jwt_key_mapping cache entries of the team's keys, issuer-scoped ones included, are gone + after the delete while entries pointing at other keys survive (LIT-5387).""" + from litellm.proxy._types import DeleteTeamRequest, LiteLLM_VerificationToken + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache + + doomed_rows: Final = ( + JWTMappingRow("hashed-doomed-key", "sub", "svc-account"), + JWTMappingRow("hashed-doomed-key", "sub", "svc-account", "https://issuer.example"), + ) + kept_row: Final = JWTMappingRow("hashed-unrelated-key", "sub", "svc-account", "https://other-issuer.example") + jwt_table: Final = CascadingJWTMappingTable([*doomed_rows, kept_row]) + + team = LiteLLM_TeamTable( + team_id="team-doomed", + team_alias="doomed-team", + members_with_roles=[], + metadata={}, + model_max_budget={}, + model_spend={}, + ) + + mock_prisma_client = AsyncMock() + mock_prisma_client.db.litellm_teamtable.find_unique = AsyncMock(return_value=team) + + async def cascading_delete_data(team_id_list, table_name): + jwt_table.cascade(("hashed-doomed-key",)) + return {"deleted_keys": 1} + + mock_prisma_client.delete_data = AsyncMock(side_effect=cascading_delete_data) + mock_prisma_client.db.litellm_deletedteamtable.create_many = AsyncMock() + mock_prisma_client.db.litellm_deletedverificationtoken.create_many = AsyncMock() + mock_prisma_client.db.litellm_verificationtoken.find_many = AsyncMock( + return_value=[LiteLLM_VerificationToken(token="hashed-doomed-key", team_id="team-doomed")] + ) + mock_prisma_client.db.litellm_jwtkeymapping = jwt_table + mock_prisma_client.db.execute_raw = AsyncMock() + mock_prisma_client.db.litellm_teammembership.delete_many = AsyncMock() + + mock_tx = AsyncMock() + mock_tx.litellm_proxymodeltable.find_many = AsyncMock(return_value=[]) + mock_tx_cm = MagicMock() + mock_tx_cm.__aenter__ = AsyncMock(return_value=mock_tx) + mock_tx_cm.__aexit__ = AsyncMock(return_value=False) + mock_prisma_client.db.tx = MagicMock(return_value=mock_tx_cm) + _wire_team_delete_tx(mock_prisma_client) + + cache: Final = UserApiKeyCache() + doomed_cache_keys: Final = _seed_jwt_mapping_cache(cache, doomed_rows) + (kept_cache_key,) = _seed_jwt_mapping_cache(cache, (kept_row,)) + + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + monkeypatch.setattr("litellm.proxy.proxy_server.user_api_key_cache", cache) + monkeypatch.setattr("litellm.proxy.proxy_server.create_audit_log_for_update", AsyncMock()) + monkeypatch.setattr("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin") + + await delete_team( + data=DeleteTeamRequest(team_ids=["team-doomed"]), + http_request=MagicMock(), + user_api_key_dict=UserAPIKeyAuth( + user_id="admin-user", api_key="sk-admin", user_role=LitellmUserRoles.PROXY_ADMIN.value + ), + litellm_changed_by="admin-user", + ) + + assert all(cache.get_cache(key=cache_key) is None for cache_key in doomed_cache_keys) + assert cache.get_cache(key=kept_cache_key) == "hashed-unrelated-key" + assert jwt_table.rows == (kept_row,) + + @pytest.mark.asyncio async def test_new_team_negative_max_budget(): """ @@ -10401,7 +10550,7 @@ def test_new_team_request_accepts_team_member_budget_duration(): async def test_create_team_member_budget_table_with_duration(): """Verify that create_team_member_budget_table passes budget_duration through to the new_budget call when team_member_budget_duration is provided.""" - from litellm.proxy._types import NewTeamRequest, UserAPIKeyAuth, LitellmUserRoles + from litellm.proxy._types import LitellmUserRoles, NewTeamRequest, UserAPIKeyAuth from litellm.proxy.management_endpoints.team_endpoints import ( TeamMemberBudgetHandler, ) @@ -10891,7 +11040,7 @@ async def test_team_member_me_matches_email_only_member(mock_db_client): @pytest.mark.asyncio async def test_team_member_me_returns_404_for_non_member(mock_db_client): """A user who is not a member of the team gets 404, regardless of role.""" - from fastapi import Request, HTTPException + from fastapi import HTTPException, Request from litellm.proxy.management_endpoints.team_endpoints import team_member_me @@ -10925,7 +11074,7 @@ async def test_team_member_me_returns_404_for_proxy_admin_not_in_team( Proxy admins get 404 if they are not actually a member of the team. `me` only resolves for actual team members; admins use /team/info instead. """ - from fastapi import Request, HTTPException + from fastapi import HTTPException, Request from litellm.proxy.management_endpoints.team_endpoints import team_member_me @@ -10986,7 +11135,7 @@ async def test_team_member_me_returns_defaults_when_no_membership_row(mock_db_cl @pytest.mark.asyncio async def test_team_member_me_rejects_team_key_without_user_id(mock_db_client): """A team key with no user_id can't resolve 'me' — must return 400.""" - from fastapi import Request, HTTPException + from fastapi import HTTPException, Request from litellm.proxy.management_endpoints.team_endpoints import team_member_me @@ -11004,7 +11153,7 @@ async def test_team_member_me_rejects_team_key_without_user_id(mock_db_client): @pytest.mark.asyncio async def test_team_member_me_returns_404_for_unknown_team(mock_db_client): """Unknown team_id returns 404 — propagated from get_team_object.""" - from fastapi import Request, HTTPException + from fastapi import HTTPException, Request from litellm.proxy.management_endpoints.team_endpoints import team_member_me diff --git a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py index fc972ccbb75..3c05068c4b0 100644 --- a/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py +++ b/tests/test_litellm/proxy/management_helpers/test_bulk_user_deletion.py @@ -8,6 +8,7 @@ import pytest from pydantic import BaseModel, ConfigDict, ValidationError from litellm.proxy._types import LiteLLM_TeamTable, LitellmUserRoles, Member, UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import jwt_key_mapping_cache_key from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.list_api.common import ManagementProblem from litellm.proxy.management_helpers.bulk_user_deletion import bulk_delete_users, bulk_remove_team_members @@ -114,6 +115,7 @@ class _Db: tokens: Sequence[Mapping[str, object]] = (), invitations: Sequence[Mapping[str, object]] = (), org_memberships: Sequence[Mapping[str, object]] = (), + jwt_mappings: Sequence[Mapping[str, object]] = (), ) -> None: self.litellm_usertable = _UserTable(users) self.litellm_teamtable = _TeamTable(teams) @@ -122,6 +124,7 @@ class _Db: self.litellm_deletedverificationtoken = _Rows() self.litellm_invitationlink = _Rows(invitations) self.litellm_organizationmembership = _Rows(org_memberships) + self.litellm_jwtkeymapping = _Rows(jwt_mappings) class _Tx: @@ -163,11 +166,12 @@ class _FakePrisma: tokens: Sequence[Mapping[str, object]] = (), invitations: Sequence[Mapping[str, object]] = (), org_memberships: Sequence[Mapping[str, object]] = (), + jwt_mappings: Sequence[Mapping[str, object]] = (), on_lock: Callable[[str], None] = lambda _: None, fail_locks: frozenset[str] = frozenset(), fail_commit: bool = False, ) -> None: - self.db = _Db(users, teams, memberships, tokens, invitations, org_memberships) + self.db = _Db(users, teams, memberships, tokens, invitations, org_memberships, jwt_mappings) self._on_lock = on_lock self._fail_locks = fail_locks self._fail_commit = fail_commit @@ -212,6 +216,17 @@ def _cache_with(*hashed_tokens: str) -> UserApiKeyCache: return cache +def _jwt_mapping(token: str, claim_value: str, issuer: str | None = None) -> Mapping[str, object]: + return {"token": token, "jwt_claim_name": "sub", "jwt_claim_value": claim_value, "jwt_issuer": issuer} + + +def _cache_with_jwt_mapping_keys(*cache_keys: str) -> UserApiKeyCache: + cache = UserApiKeyCache() + for key in cache_keys: + cache.set_cache(key=key, value={"cache_key": key}) + return cache + + async def _delete( prisma: _FakePrisma, user_ids: Sequence[str], @@ -449,6 +464,34 @@ async def test_bulk_delete_evicts_deleted_keys_and_users_from_the_auth_cache(): assert cache.get_cache(key="keep-key") is not None +@pytest.mark.asyncio +async def test_bulk_delete_evicts_jwt_key_mappings_of_the_deleted_users_keys(): + issuer: Final = "https://issuer.example" + doomed_global: Final = jwt_key_mapping_cache_key("sub", "alice") + doomed_scoped: Final = jwt_key_mapping_cache_key("sub", "alice", issuer) + kept: Final = jwt_key_mapping_cache_key("sub", "bob") + prisma = _FakePrisma( + users=[_user("u1", "t1"), _user("keep", "t1")], + teams=[_team("t1", "u1", "keep")], + tokens=[ + {"token": "team-key", "user_id": "u1", "team_id": "t1"}, + {"token": "personal-key", "user_id": "u1"}, + {"token": "keep-key", "user_id": "keep", "team_id": "t1"}, + ], + jwt_mappings=[ + _jwt_mapping("personal-key", "alice"), + _jwt_mapping("team-key", "alice", issuer=issuer), + _jwt_mapping("keep-key", "bob"), + ], + ) + cache = _cache_with_jwt_mapping_keys(doomed_global, doomed_scoped, kept) + + await _delete(prisma, ["u1"], cache=cache) + + assert cache.get_cache(key=doomed_global) is None and cache.get_cache(key=doomed_scoped) is None + assert cache.get_cache(key=kept) is not None + + @pytest.mark.asyncio async def test_bulk_delete_rejects_non_admin_callers_before_touching_the_db(): prisma = _FakePrisma(users=[_user("u1")]) @@ -577,6 +620,28 @@ async def test_bulk_member_delete_evicts_the_removed_team_keys_from_the_auth_cac assert cache.get_cache(key="keep-key") is not None +@pytest.mark.asyncio +async def test_bulk_member_delete_evicts_jwt_key_mappings_of_the_removed_team_keys(): + issuer: Final = "https://issuer.example" + doomed: Final = jwt_key_mapping_cache_key("sub", "alice", issuer) + kept: Final = jwt_key_mapping_cache_key("sub", "bob") + prisma = _FakePrisma( + users=[_user("u1", "t1"), _user("keep", "t1")], + teams=[_team("t1", "u1", "keep")], + tokens=[ + {"token": "team-key", "user_id": "u1", "team_id": "t1"}, + {"token": "keep-key", "user_id": "keep", "team_id": "t1"}, + ], + jwt_mappings=[_jwt_mapping("team-key", "alice", issuer=issuer), _jwt_mapping("keep-key", "bob")], + ) + cache = _cache_with_jwt_mapping_keys(doomed, kept) + + await _remove(prisma, "t1", [{"user_id": "u1"}], cache=cache) + + assert cache.get_cache(key=doomed) is None + assert cache.get_cache(key=kept) is not None + + @pytest.mark.asyncio async def test_bulk_member_delete_cleans_a_user_whose_teams_array_still_names_the_team(): prisma = _FakePrisma(users=[_user("stale", "t1")], teams=[_team("t1", "other")], memberships=[("t1", "stale")]) diff --git a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py index 48eeb39fecf..42cd6e4ed78 100644 --- a/tests/test_litellm/proxy/proxy_server/test_proxy_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_proxy_config.py @@ -134,20 +134,14 @@ def test__scrub_db_overlay_remote_module_loads_invalid_non_dict_returns_input(): def test_resolve_complexity_router_plugins_no_plugins_key_is_a_noop(): config: Dict[str, Any] = {"tiers": {"SIMPLE": "gpt-4o-mini"}} - resolve_complexity_router_plugins( - model_name="smart-router", complexity_router_config=config, config_file_path=None - ) + resolve_complexity_router_plugins(model_name="smart-router", complexity_router_config=config, config_file_path=None) assert config == {"tiers": {"SIMPLE": "gpt-4o-mini"}} def test_resolve_complexity_router_plugins_resolves_dotted_path_to_live_instance(tmp_path): plugin_file = tmp_path / "my_plugin.py" plugin_file.write_text( - "class _Plugin:\n" - " async def run(self, context):\n" - " return context\n" - "\n" - "my_plugin_instance = _Plugin()\n" + "class _Plugin:\n async def run(self, context):\n return context\n\nmy_plugin_instance = _Plugin()\n" ) config: Dict[str, Any] = {"plugins": ["my_plugin.my_plugin_instance"]} @@ -262,9 +256,18 @@ def _custom_prompt_row(model_name: str) -> dict[str, object]: [ ([_heuristic_v2_row("a"), _heuristic_v2_row("b"), _heuristic_v2_row("c", "heuristic")], "heuristic_v2"), ([_custom_tier_row("a"), _custom_tier_row("b"), _heuristic_v2_row("c", "heuristic")], "tier_definitions"), - ([_custom_prompt_row("a"), _custom_prompt_row("b"), _heuristic_v2_row("c", "heuristic")], "operator-written classifier prompt"), - ([_custom_tier_row("a"), _custom_prompt_row("b"), _heuristic_v2_row("c", "heuristic")], "operator-written classifier prompt"), - ([_operator_examples_row("a"), _custom_tier_row("b"), _heuristic_v2_row("c", "heuristic")], "operator-written classifier prompt"), + ( + [_custom_prompt_row("a"), _custom_prompt_row("b"), _heuristic_v2_row("c", "heuristic")], + "operator-written classifier prompt", + ), + ( + [_custom_tier_row("a"), _custom_prompt_row("b"), _heuristic_v2_row("c", "heuristic")], + "operator-written classifier prompt", + ), + ( + [_operator_examples_row("a"), _custom_tier_row("b"), _heuristic_v2_row("c", "heuristic")], + "operator-written classifier prompt", + ), ], ) def test_validate_auto_router_capability_limits_refuses_to_start_over_the_limit( @@ -339,20 +342,17 @@ async def test_ProxyConfig_load_config_takes_the_classifier_limit_from_the_licen ), } config_yaml = _TWO_HEURISTIC_V2_ROUTERS_YAML.replace( - "classifier_type: heuristic_v2\n", f"classifier_type: {classifier_type}\n{forecast_settings.get(classifier_type, '')}" + "classifier_type: heuristic_v2\n", + f"classifier_type: {classifier_type}\n{forecast_settings.get(classifier_type, '')}", ).replace("tiers: {SIMPLE: gpt-4o-mini}", "tiers: {SIMPLE: gpt-4o-mini, REASONING: gpt-4o}") f.write_text(config_yaml) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) - monkeypatch.setattr( - "litellm.proxy.proxy_server._license_check.auto_router_capability_limit", lambda: license_limit - ) + monkeypatch.setattr("litellm.proxy.proxy_server._license_check.auto_router_capability_limit", lambda: license_limit) if license_limit is None: - router, _model_list, _general_settings = await ProxyConfig().load_config( - router=None, config_file_path=str(f) - ) + router, _model_list, _general_settings = await ProxyConfig().load_config(router=None, config_file_path=str(f)) assert router.auto_router_capability_limit is not None assert router.auto_router_capability_limit() is None assert sorted(router.complexity_routers) == ["v2-a", "v2-b"] @@ -371,10 +371,12 @@ async def test_ProxyConfig_load_config_router_refuses_a_db_heuristic_v2_router_b from litellm.types.router import Deployment f = tmp_path / "c.yaml" - f.write_text(_TWO_HEURISTIC_V2_ROUTERS_YAML.replace(" - model_name: v2-b\n", " - model_name: v1-b\n", 1).replace( - "classifier_type: heuristic_v2\n tiers: {SIMPLE: gpt-4o-mini}\nrouter_settings", - "classifier_type: heuristic\n tiers: {SIMPLE: gpt-4o-mini}\nrouter_settings", - )) + f.write_text( + _TWO_HEURISTIC_V2_ROUTERS_YAML.replace(" - model_name: v2-b\n", " - model_name: v1-b\n", 1).replace( + "classifier_type: heuristic_v2\n tiers: {SIMPLE: gpt-4o-mini}\nrouter_settings", + "classifier_type: heuristic\n tiers: {SIMPLE: gpt-4o-mini}\nrouter_settings", + ) + ) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) @@ -557,9 +559,7 @@ def test_resolve_complexity_router_plugins_leaves_live_classifier_instance_alone instance = _Classifier() config: dict[str, Any] = {"classifier_plugin": instance} - resolve_complexity_router_plugins( - model_name="smart-router", complexity_router_config=config, config_file_path=None - ) + resolve_complexity_router_plugins(model_name="smart-router", complexity_router_config=config, config_file_path=None) assert config["classifier_plugin"] is instance @@ -571,11 +571,7 @@ def test_resolve_complexity_router_plugins_leaves_live_classifier_instance_alone def test_resolve_routing_plugins_resolves_dotted_paths(tmp_path): plugin_file = tmp_path / "rs_plugin.py" plugin_file.write_text( - "class _Plugin:\n" - " async def run(self, context):\n" - " return context\n" - "\n" - "rs_plugin_instance = _Plugin()\n" + "class _Plugin:\n async def run(self, context):\n return context\n\nrs_plugin_instance = _Plugin()\n" ) resolved = resolve_routing_plugins( @@ -1527,7 +1523,7 @@ async def test_ProxyConfig_get_config_missing_file_raises(monkeypatch): # ProxyConfig._initialize_secret_manager_from_raw_config # --------------------------------------------------------------------------- -VAULT_SECRET_MANAGER_MODULE = ''' +VAULT_SECRET_MANAGER_MODULE = """ import os from litellm.integrations.custom_secret_manager import CustomSecretManager @@ -1548,7 +1544,7 @@ class VaultSecretManager(CustomSecretManager): async def async_read_secret(self, secret_name, optional_params=None, timeout=None, **kwargs): return VAULT.get(secret_name) -''' +""" VAULT_BACKED_CONFIG = """ model_list: @@ -1649,9 +1645,7 @@ async def test_ProxyConfig_get_config_reuses_an_already_initialized_secret_manag @pytest.mark.asyncio -async def test_ProxyConfig_get_config_without_key_management_system_leaves_secret_manager_unset( - tmp_path, monkeypatch -): +async def test_ProxyConfig_get_config_without_key_management_system_leaves_secret_manager_unset(tmp_path, monkeypatch): """No ``key_management_system`` means no manager, an unresolvable reference stays None, and nothing is warned about: with no manager there is nothing to have been absent from.""" config_yaml = VAULT_BACKED_CONFIG.replace(" key_management_system: custom\n", "") @@ -1670,9 +1664,7 @@ async def test_ProxyConfig_get_config_without_key_management_system_leaves_secre @pytest.mark.asyncio -async def test_ProxyConfig_get_config_warns_when_a_reference_is_missing_from_the_secret_manager( - tmp_path, monkeypatch -): +async def test_ProxyConfig_get_config_warns_when_a_reference_is_missing_from_the_secret_manager(tmp_path, monkeypatch): """A reference the manager cannot resolve is logged, instead of silently becoming None.""" config_yaml = VAULT_BACKED_CONFIG.replace("MY_PROVIDER_KEY", "NOT_IN_VAULT") config_file_path = _write_vault_backed_config(tmp_path, monkeypatch, config_yaml) @@ -2109,10 +2101,7 @@ async def test_ProxyConfig_load_config_minimal_yaml(tmp_path, monkeypatch): async def test_load_config_logs_disabled_budget_reservation_once(tmp_path, monkeypatch, caplog, setting): config_file = tmp_path / "budget.yaml" flag = f" disable_budget_reservation: {setting}\n" if setting is not None else "" - config_file.write_text( - "model_list: []\nlitellm_settings: {}\ngeneral_settings:\n" - " master_key: null\n" + flag - ) + config_file.write_text("model_list: []\nlitellm_settings: {}\ngeneral_settings:\n master_key: null\n" + flag) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) monkeypatch.setattr("litellm.constants.budget_reservation_disabled_info_emitted", False) @@ -2123,10 +2112,7 @@ async def test_load_config_logs_disabled_budget_reservation_once(tmp_path, monke for _ in range(3): await config.load_config(router=None, config_file_path=str(config_file)) - records = [ - record for record in caplog.records - if "disable_budget_reservation is enabled" in record.message - ] + records = [record for record in caplog.records if "disable_budget_reservation is enabled" in record.message] assert [record.levelno for record in records] == ([logging.INFO] if setting == "true" else []) @@ -2138,11 +2124,7 @@ async def test_ProxyConfig_load_config_resolves_router_settings_plugins(tmp_path to `await "some.string".run(context)`.""" plugin_file = tmp_path / "rs_plugin.py" plugin_file.write_text( - "class _Plugin:\n" - " async def run(self, context):\n" - " return context\n" - "\n" - "rs_plugin_instance = _Plugin()\n" + "class _Plugin:\n async def run(self, context):\n return context\n\nrs_plugin_instance = _Plugin()\n" ) f = tmp_path / "c.yaml" f.write_text( @@ -2157,9 +2139,7 @@ async def test_ProxyConfig_load_config_resolves_router_settings_plugins(tmp_path monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) - router, _model_list, _general_settings = await ProxyConfig().load_config( - router=None, config_file_path=str(f) - ) + router, _model_list, _general_settings = await ProxyConfig().load_config(router=None, config_file_path=str(f)) assert len(router.routing_plugins) == 1 assert type(router.routing_plugins[0]).__name__ == "_Plugin" @@ -2226,10 +2206,7 @@ async def test_ProxyConfig_load_config_wires_config_reload_interval(tmp_path, mo f = tmp_path / "c.yaml" f.write_text( - "model_list: []\n" - "general_settings:\n" - " proxy_config_reload_interval_seconds: 47\n" - "litellm_settings: {}\n" + "model_list: []\ngeneral_settings:\n proxy_config_reload_interval_seconds: 47\nlitellm_settings: {}\n" ) monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) @@ -2371,13 +2348,9 @@ async def test_ProxyConfig__init_non_llm_configs_premium_invalid_worker_registry async def test_ProxyConfig__init_non_llm_configs_worker_registry_requires_premium(monkeypatch): monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", False) pc = ProxyConfig() - with pytest.raises(ValueError, match='Trying to use `worker_registry`You must be a LiteLLM') as exc_info: + with pytest.raises(ValueError, match="Trying to use `worker_registry`You must be a LiteLLM") as exc_info: await pc._init_non_llm_configs( - config={ - "worker_registry": [ - {"worker_id": "worker-a", "name": "Worker A", "url": "http://localhost:4001"} - ] - }, + config={"worker_registry": [{"worker_id": "worker-a", "name": "Worker A", "url": "http://localhost:4001"}]}, config_file_path=None, ) message = str(exc_info.value) @@ -2607,9 +2580,7 @@ def test_ProxyConfig__warn_on_misplaced_jwt_keys_warns_even_when_also_under_gene def test_ProxyConfig__warn_on_misplaced_jwt_keys_silent_when_correctly_placed(): """Keys living only under general_settings are valid, so no warning fires.""" - result, warnings = _capture_proxy_warnings( - {"general_settings": {"enable_jwt_auth": True, "litellm_jwtauth": {}}} - ) + result, warnings = _capture_proxy_warnings({"general_settings": {"enable_jwt_auth": True, "litellm_jwtauth": {}}}) assert result == () assert warnings == [] @@ -2636,7 +2607,7 @@ def test_ProxyConfig_initialize_secret_manager_none_noop(): def test_ProxyConfig_initialize_secret_manager_invalid_kms_raises(): pc = ProxyConfig() - with pytest.raises(ValueError, match='Invalid Key Management System selected'): + with pytest.raises(ValueError, match="Invalid Key Management System selected"): pc.initialize_secret_manager(key_management_system="not-a-real-kms") @@ -3141,28 +3112,6 @@ async def test_ProxyConfig__update_llm_router_no_models_smoke(monkeypatch): assert snapshot == {"raised": False, "called": True, "models": "empty"} -@pytest.mark.asyncio -async def test_ProxyConfig__update_llm_router_bad_proxy_logging_raises(monkeypatch): - pc = ProxyConfig() - - async def fake_get_config(): - # alerting present + non-list general_settings to trigger the alerting branch. - return {"general_settings": {"alerting": ["slack"]}} - - fake_router = MagicMock() - fake_router.update_settings = MagicMock() - monkeypatch.setattr(pc, "get_config", fake_get_config) - monkeypatch.setattr("litellm.proxy.proxy_server.llm_router", fake_router) - monkeypatch.setattr("litellm.proxy.proxy_server.master_key", "sk-x") - monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) - monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {"alerting": ["email"]}) - monkeypatch.setattr("litellm.proxy.proxy_server.proxy_config", pc) - # Passing None for proxy_logging_obj triggers AttributeError in _add_general_settings_from_db_config - # when it calls proxy_logging_obj.update_values. - with pytest.raises(AttributeError): - await pc._update_llm_router(new_models=[], proxy_logging_obj=None) # type: ignore[arg-type] - - # --------------------------------------------------------------------------- # ProxyConfig._add_callback_from_db_to_in_memory_litellm_callbacks # --------------------------------------------------------------------------- @@ -3637,43 +3586,6 @@ async def test_ProxyConfig_get_credentials_reads_from_writer_not_replica(monkeyp reader_inner.litellm_credentialstable.find_many.assert_not_awaited() -# --------------------------------------------------------------------------- -# ProxyConfig._add_general_settings_from_db_config -# --------------------------------------------------------------------------- - - -def test_ProxyConfig__add_general_settings_from_db_config_merges_alerting(): - pc = ProxyConfig() - proxy_logging = MagicMock() - general = {"alerting": ["slack"]} - config_data = {"general_settings": {"alerting": ["email", "slack"]}} - pc._add_general_settings_from_db_config( - config_data=config_data, - general_settings=general, - proxy_logging_obj=proxy_logging, - ) - snapshot = { - "alerting": sorted(general["alerting"]), - "logging_called": proxy_logging.update_values.called, - "merged_count": len(general["alerting"]), - } - assert snapshot == { - "alerting": ["email", "slack"], - "logging_called": True, - "merged_count": 2, - } - - -def test_ProxyConfig__add_general_settings_from_db_config_bad_config_raises(): - pc = ProxyConfig() - with pytest.raises(AttributeError): - pc._add_general_settings_from_db_config( - config_data=None, # type: ignore[arg-type] - general_settings={}, - proxy_logging_obj=MagicMock(), - ) - - # --------------------------------------------------------------------------- # ProxyConfig._reschedule_spend_log_cleanup_job # --------------------------------------------------------------------------- @@ -3736,7 +3648,9 @@ async def test_ProxyConfig__update_general_settings_updates_health_check_retenti reschedule = AsyncMock() monkeypatch.setattr(pc, "_reschedule_spend_log_cleanup_job", reschedule) await pc._update_general_settings({"maximum_health_check_retention_period": "30d"}) - assert settings["maximum_health_check_retention_period"] == "30d" + from litellm.proxy import proxy_server + + assert proxy_server.general_settings["maximum_health_check_retention_period"] == "30d" reschedule.assert_awaited_once() @@ -3790,7 +3704,6 @@ async def test_ProxyConfig__update_general_settings_yaml_max_batch_file_size_mb_ {"max_batch_file_size_mb": 3}, ) pc = ProxyConfig() - pc._yaml_general_settings_keys = {"max_batch_file_size_mb"} await pc._update_general_settings({"max_batch_file_size_mb": 5}) from litellm.proxy import proxy_server as ps @@ -3807,7 +3720,7 @@ async def test_ProxyConfig__update_general_settings_cleared_db_max_batch_file_si await pc._update_general_settings({"max_parallel_requests": 1}) from litellm.proxy import proxy_server as ps - assert ps.general_settings.get("max_batch_file_size_mb") is None + assert ps.general_settings.get("max_batch_file_size_mb") == 8 @pytest.mark.asyncio @@ -3827,7 +3740,6 @@ async def test_ProxyConfig__update_general_settings_yaml_allowed_file_extensions {"allowed_file_extensions": [".pdf"]}, ) pc = ProxyConfig() - pc._yaml_general_settings_keys = {"allowed_file_extensions"} await pc._update_general_settings({"allowed_file_extensions": [".jsonl"]}) from litellm.proxy import proxy_server as ps @@ -3845,27 +3757,195 @@ async def test_ProxyConfig__update_general_settings_none_input_noop(): await pc._update_general_settings(db_general_settings=12345) # type: ignore[arg-type] -# --------------------------------------------------------------------------- -# ProxyConfig._update_config_fields -# --------------------------------------------------------------------------- +@pytest.mark.asyncio +async def test_ProxyConfig__update_general_settings_skips_redundant_retention_reschedule(monkeypatch): + from litellm.proxy import proxy_server - -def test_ProxyConfig__update_config_fields_merges_dict(): pc = ProxyConfig() - current = {"general_settings": {"a": 1, "b": 2}} - out = pc._update_config_fields( - current_config=current, - param_name="general_settings", - db_param_value={"b": 3, "c": 4, "d": 5}, + reschedule: Final = AsyncMock() + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(pc, "_reschedule_spend_log_cleanup_job", reschedule) + + await pc._update_general_settings({"maximum_health_check_retention_period": "30d"}) + reschedule.assert_awaited_once() + reschedule.reset_mock() + + await pc._update_general_settings({"maximum_health_check_retention_period": "30d"}) + + reschedule.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_ProxyConfig__update_general_settings_reschedules_after_retention_key_deletion(monkeypatch): + from litellm.proxy import proxy_server + + pc = ProxyConfig() + reschedule: Final = AsyncMock() + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(pc, "_reschedule_spend_log_cleanup_job", reschedule) + + await pc._update_general_settings({"maximum_health_check_retention_period": "30d"}) + reschedule.reset_mock() + + await pc._update_general_settings({}) + + reschedule.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_ProxyConfig__update_general_settings_dispatches_every_side_effect_handler(monkeypatch): + pc = ProxyConfig() + handlers: Final = ( + ("_apply_alerting_settings", AsyncMock()), + ("_apply_pass_through_settings", AsyncMock()), + ("_apply_boolean_settings", AsyncMock()), + ("_apply_store_model_in_db_setting", AsyncMock()), + ("_apply_retention_settings", AsyncMock()), + ("_apply_ssrf_settings", AsyncMock()), + ("_apply_cache_size_setting", AsyncMock()), ) - assert out == {"general_settings": {"a": 1, "b": 3, "c": 4, "d": 5}} + for name, handler in handlers: + monkeypatch.setattr(pc, name, handler) + + await pc._apply_general_settings_side_effects({}, False, (), None) + + for name, handler in handlers: + if name == "_apply_cache_size_setting": + handler.assert_awaited_once_with({}, cache_size_was_db=False) + elif name == "_apply_retention_settings": + handler.assert_awaited_once_with({}, previous_retention_values=()) + elif name == "_apply_pass_through_settings": + handler.assert_awaited_once_with({}, previous_endpoints=None) + else: + handler.assert_awaited_once_with({}) -def test_ProxyConfig__update_config_fields_invalid_param_raises(): +@pytest.mark.asyncio +async def test_ProxyConfig__update_general_settings_unrelated_value_fires_no_runtime_effect(monkeypatch): + from litellm.proxy import proxy_server + pc = ProxyConfig() - with pytest.raises(TypeError): - # Missing required arg. - pc._update_config_fields(current_config={}, param_name="general_settings") # type: ignore[call-arg] + initialize_endpoints: Final = AsyncMock() + reschedule: Final = AsyncMock() + cache: Final = MagicMock() + proxy_logging: Final = MagicMock() + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "initialize_pass_through_endpoints", initialize_endpoints) + monkeypatch.setattr(proxy_server, "user_api_key_cache", cache) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", proxy_logging) + monkeypatch.setattr(pc, "_reschedule_spend_log_cleanup_job", reschedule) + + await pc._update_general_settings({"unrelated": "value"}) + + initialize_endpoints.assert_not_awaited() + reschedule.assert_not_awaited() + cache.update_in_memory_max_size.assert_not_called() + proxy_logging.update_values.assert_not_called() + proxy_logging.slack_alerting_instance.update_values.assert_not_called() + + +@pytest.mark.asyncio +async def test_ProxyConfig__update_config_from_db_resolves_through_settings_stores(monkeypatch): + pc = ProxyConfig() + config = { + "general_settings": { + "max_file_size_mb": 7, + "max_parallel_requests": 3, + "alerting": ["config"], + "pass_through_endpoints": [{"path": "/config"}], + "maximum_spend_logs_cleanup_batch_size": 10, + }, + "router_settings": {"fallbacks": ["config"], "num_retries": 1}, + } + db_values = { + "general_settings": { + "max_file_size_mb": 9, + "max_parallel_requests": 11, + "alerting": ["db"], + "pass_through_endpoints": [{"path": "/db"}], + "maximum_spend_logs_cleanup_batch_size": None, + }, + "router_settings": {"fallbacks": [], "num_retries": 2}, + } + + async def get_config_param(_, param_name): + value = db_values.get(param_name) + return SimpleNamespace(param_name=param_name, param_value=value) if value is not None else None + + monkeypatch.setattr("litellm.proxy.proxy_server.get_config_param", get_config_param) + pc._load_yaml_settings_stores(config) + + resolved = await pc._update_config_from_db(MagicMock(), config, store_model_in_db=True) + + assert resolved["general_settings"] == { + "max_file_size_mb": 7, + "max_parallel_requests": 3, + "alerting": ["config"], + "pass_through_endpoints": [{"path": "/config"}], + "maximum_spend_logs_cleanup_batch_size": 10, + } + assert resolved["router_settings"] == {"fallbacks": ["config"], "num_retries": 1} + assert pc.settings.source("max_file_size_mb") == "config" + assert pc.settings.source("max_parallel_requests") == "config" + + +@pytest.mark.asyncio +async def test_ProxyConfig__update_config_from_db_keeps_keys_the_config_file_omits(monkeypatch): + pc = ProxyConfig() + config = {"general_settings": {"max_file_size_mb": 7}, "router_settings": {"num_retries": 1}} + db_values = { + "general_settings": {"max_file_size_mb": 9, "max_parallel_requests": 11}, + "router_settings": {"fallbacks": ["db"], "num_retries": 2}, + } + + async def get_config_param(_, param_name): + value = db_values.get(param_name) + return SimpleNamespace(param_name=param_name, param_value=value) if value is not None else None + + monkeypatch.setattr("litellm.proxy.proxy_server.get_config_param", get_config_param) + pc._load_yaml_settings_stores(config) + + resolved = await pc._update_config_from_db(MagicMock(), config, store_model_in_db=True) + + assert resolved["general_settings"] == {"max_file_size_mb": 7, "max_parallel_requests": 11} + assert resolved["router_settings"] == {"num_retries": 1, "fallbacks": ["db"]} + assert pc.settings.source("max_parallel_requests") == "db" + + +def test_ProxyConfig_load_yaml_settings_stores_keeps_db_endpoints_out_of_config_baseline(): + from litellm.proxy import proxy_server + + pc = ProxyConfig() + config_endpoint: Final = {"path": "/config", "target": "https://config.example"} + db_endpoint: Final = {"id": "db-endpoint", "path": "/db", "target": "https://db.example"} + + pc._load_yaml_settings_stores({"general_settings": {"pass_through_endpoints": [config_endpoint]}}) + pc.settings.apply_db_row("general_settings", {"pass_through_endpoints": [db_endpoint]}) + + assert proxy_server.config_passthrough_endpoints == [config_endpoint] + + +@pytest.mark.asyncio +async def test_ProxyConfig_add_deployment_continues_after_null_pass_through_endpoints(monkeypatch): + from litellm.proxy import proxy_server + + pc = ProxyConfig() + non_llm_initialization = AsyncMock() + monkeypatch.setattr(proxy_server, "general_settings", {}) + monkeypatch.setattr(proxy_server, "prefetch_config_params", AsyncMock()) + monkeypatch.setattr( + proxy_server, + "get_config_param", + AsyncMock(return_value=SimpleNamespace(param_value={"pass_through_endpoints": None})), + ) + monkeypatch.setattr(proxy_server, "sync_ui_settings_to_general_settings", AsyncMock()) + monkeypatch.setattr(pc, "_should_load_db_object", lambda *, object_type: False) + monkeypatch.setattr(pc, "get_credentials", AsyncMock()) + monkeypatch.setattr(pc, "_init_non_llm_objects_in_db", non_llm_initialization) + + await pc.add_deployment(prisma_client=MagicMock(), proxy_logging_obj=MagicMock()) + + non_llm_initialization.assert_awaited_once() # --------------------------------------------------------------------------- diff --git a/tests/test_litellm/proxy/proxy_server/test_routes_config.py b/tests/test_litellm/proxy/proxy_server/test_routes_config.py index dd3914e3ad5..19b36b026f9 100644 --- a/tests/test_litellm/proxy/proxy_server/test_routes_config.py +++ b/tests/test_litellm/proxy/proxy_server/test_routes_config.py @@ -22,6 +22,18 @@ import pytest from .conftest import VOLATILE_KEYS, normalize +def _seed_settings_store(monkeypatch, db_row: dict, yaml_values: dict | None = None) -> None: + """Point proxy_config.settings at a store holding the same row the mocked table returns, + the way a booted proxy does, so the read routes resolve against it.""" + from litellm.proxy import proxy_server as ps + from litellm.proxy.config_resolvers import SettingsStore + + store = SettingsStore("general_settings") + store.load_yaml(yaml_values or {}) + store.apply_db_row("general_settings", db_row) + monkeypatch.setattr(ps.proxy_config, "settings", store) + + def _install_litellm_config(mock_prisma: MagicMock) -> MagicMock: """Ensure mock_prisma.db.litellm_config exists with async methods (the conftest only stubs ``litellm_configtable`` — this is a different table).""" @@ -322,7 +334,7 @@ def test_config_field_update_invalid_field(client, auth_as, mock_prisma, monkeyp def test_config_field_info_happy_admin(client, auth_as, mock_prisma, monkeypatch): - """Admin gets back ConfigFieldInfo with the stored value pulled from DB.""" + """Admin gets back ConfigFieldInfo with the value the proxy resolved, tagged with where it came from.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -331,6 +343,7 @@ def test_config_field_info_happy_admin(client, auth_as, mock_prisma, monkeypatch row.param_value = {"max_parallel_requests": 7} table.find_first = AsyncMock(return_value=row) monkeypatch.setattr(ps, "prisma_client", mock_prisma) + _seed_settings_store(monkeypatch, row.param_value) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/config/field/info", params={"field_name": "max_parallel_requests"}) @@ -338,6 +351,8 @@ def test_config_field_info_happy_admin(client, auth_as, mock_prisma, monkeypatch assert normalize(response.json()) == { "field_name": "max_parallel_requests", "field_value": 7, + "source": "db", + "editable": True, } @@ -356,7 +371,7 @@ def test_config_field_info_non_admin_rejected(client, auth_as, mock_prisma, monk def test_config_field_info_field_not_in_db(client, auth_as, mock_prisma, monkeypatch): - """When the field is missing from the DB row, returns 400 'not in DB'.""" + """When nothing sets the field, neither the config file nor the DB row, it 400s.""" from litellm.proxy import proxy_server as ps from litellm.proxy._types import LitellmUserRoles @@ -365,11 +380,12 @@ def test_config_field_info_field_not_in_db(client, auth_as, mock_prisma, monkeyp row.param_value = {"some_other_field": "value"} table.find_first = AsyncMock(return_value=row) monkeypatch.setattr(ps, "prisma_client", mock_prisma) + _seed_settings_store(monkeypatch, row.param_value) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/config/field/info", params={"field_name": "max_parallel_requests"}) assert response.status_code == 400 - assert "not in DB" in response.json().get("detail", {}).get("error", "") + assert "is not set" in response.json().get("detail", {}).get("error", "") def test_config_field_info_redacts_nested_secret_for_view_only_admin(client, auth_as, mock_prisma, monkeypatch): @@ -391,6 +407,7 @@ def test_config_field_info_redacts_nested_secret_for_view_only_admin(client, aut } table.find_first = AsyncMock(return_value=row) monkeypatch.setattr(ps, "prisma_client", mock_prisma) + _seed_settings_store(monkeypatch, row.param_value) with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): response = client.get("/config/field/info", params={"field_name": "database_args"}) @@ -417,6 +434,7 @@ def test_config_field_info_full_admin_sees_nested_secret(client, auth_as, mock_p } table.find_first = AsyncMock(return_value=row) monkeypatch.setattr(ps, "prisma_client", mock_prisma) + _seed_settings_store(monkeypatch, row.param_value) with auth_as(LitellmUserRoles.PROXY_ADMIN): response = client.get("/config/field/info", params={"field_name": "database_args"}) @@ -438,6 +456,7 @@ def test_config_field_info_redacts_top_level_scalar_for_view_only(client, auth_a row.param_value = {"database_url": "postgresql://admin:p4ss@db:5432/litellm"} table.find_first = AsyncMock(return_value=row) monkeypatch.setattr(ps, "prisma_client", mock_prisma) + _seed_settings_store(monkeypatch, row.param_value) with auth_as(LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): response = client.get("/config/field/info", params={"field_name": "database_url"}) diff --git a/tests/test_litellm/proxy/test_plugin_routes.py b/tests/test_litellm/proxy/test_plugin_routes.py index 52999447179..c8d2939385d 100644 --- a/tests/test_litellm/proxy/test_plugin_routes.py +++ b/tests/test_litellm/proxy/test_plugin_routes.py @@ -14,6 +14,8 @@ Covers three bugs: import asyncio from unittest.mock import MagicMock +import pytest + from litellm.proxy._types import ( ConfigGeneralSettings, LitellmUserRoles, @@ -131,16 +133,16 @@ def test_plugin_key_is_never_returned_to_the_browser() -> None: register_plugins_from_config({}) -def test_db_persisted_plugins_load_on_startup() -> None: - """Plugins saved to DB general_settings must register when the DB config is - merged at startup, not just when present in the YAML file.""" +def test_db_persisted_plugins_load_on_startup(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.proxy import proxy_server from litellm.proxy.proxy_server import ProxyConfig - register_plugins_from_config({}) # start empty (as if YAML had no plugins) + register_plugins_from_config({}) + monkeypatch.setattr(proxy_server, "general_settings", {}) - ProxyConfig()._add_general_settings_from_db_config( - config_data={ - "general_settings": { + asyncio.run( + ProxyConfig()._update_general_settings( + { "plugins": [ { "name": "db-plugin", @@ -149,9 +151,7 @@ def test_db_persisted_plugins_load_on_startup() -> None: } ] } - }, - general_settings={}, - proxy_logging_obj=MagicMock(), + ) ) names = [p["name"] for p in asyncio.run(list_plugins(user_api_key_dict=_admin()))] diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 2faaf832d93..82565f000bf 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -9,6 +9,7 @@ import socket import subprocess import time import types +import uuid from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Final @@ -1087,7 +1088,9 @@ async def test_init_mcp_servers_from_db_respects_supported_db_objects(monkeypatc mock_init.assert_not_awaited() -def test_update_config_fields_deep_merge_db_wins(): +def test_settings_store_deep_merge_db_wins(): + """The config file owns model_group_alias outright once it declares it, so a stored + row can no longer add, replace or partially update entries inside it.""" from litellm.proxy.proxy_server import ProxyConfig proxy_config = ProxyConfig() @@ -1127,29 +1130,15 @@ def test_update_config_fields_deep_merge_db_wins(): } } - updated = proxy_config._update_config_fields( - current_config=current_config, - param_name="router_settings", - db_param_value=db_param_value, - ) + proxy_config.router_settings.load_yaml(current_config["router_settings"]) + proxy_config.router_settings.apply_db_row("router_settings", db_param_value) - rs = updated["router_settings"] + rs = proxy_config.router_settings.resolved() aliases = rs["model_group_alias"] - # DB wins on conflicts (deep) for existing alias - assert aliases["claude-sonnet-4"]["model"] == "claude-sonnet-4-20250514" - assert aliases["claude-sonnet-4"]["hidden"] is False - - # New alias introduced by DB is present with its values - assert "claude-sonnet-latest" in aliases - assert aliases["claude-sonnet-latest"]["model"] == "claude-sonnet-4-20250514" - assert aliases["claude-sonnet-latest"]["hidden"] is True - - # None in DB does not overwrite existing values - assert aliases["legacy-sonnet"]["model"] == "claude-2.1" - assert aliases["legacy-sonnet"]["hidden"] is True - - # Unrelated router_settings keys are preserved + assert aliases == current_config["router_settings"]["model_group_alias"] + assert "claude-sonnet-latest" not in aliases + assert proxy_config.router_settings.source("model_group_alias") == "config" assert rs["routing_mode"] == "cost_optimized" @@ -4946,26 +4935,14 @@ async def test_add_router_settings_from_db_config_merge_logic(): call_args = mock_router.update_settings.call_args combined_settings = call_args[1] # kwargs - # Verify the merge results - # DB values should override config values - assert combined_settings["routing_strategy"] == "least-busy" - - # Config-only values should be preserved + assert combined_settings["routing_strategy"] == "usage-based-routing" assert combined_settings["model_group_alias"] == {"gpt-4": "openai-gpt-4"} - assert combined_settings["enable_pre_call_checks"] == True + assert combined_settings["enable_pre_call_checks"] is True assert combined_settings["timeout"] == 30 + assert combined_settings["nested_config"] == {"setting1": "config_value1", "setting2": "config_value2"} - # DB-only values should be added assert combined_settings["retry_delay"] == 2 - # Nested dictionaries should be merged (but this is shallow merge) - expected_nested = { - "setting1": "config_value1", - "setting2": "db_value2", - "setting3": "db_value3", - } - assert combined_settings["nested_config"] == expected_nested - def _routing_groups_router(): from litellm import Router @@ -5077,7 +5054,7 @@ async def test_add_router_settings_from_db_config_empty_db_lists_do_not_clobber_ combined_settings = mock_router.update_settings.call_args.kwargs assert combined_settings["fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] assert combined_settings["context_window_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] - assert combined_settings["content_policy_fallbacks"] == [{"gpt-oss-120b": ["other-model"]}] + assert combined_settings["content_policy_fallbacks"] == [{"gpt-oss-120b": ["granite-4-h-small"]}] assert combined_settings["num_retries"] == 3 @@ -5264,8 +5241,8 @@ async def test_add_router_settings_shallow_merge_behavior(): "key4": "db_value4", } - assert merged_settings["nested_setting"] == expected_nested - assert merged_settings["top_level"] == "db_top" + assert merged_settings["nested_setting"] == config_data["router_settings"]["nested_setting"] + assert merged_settings["top_level"] == "config_top" @pytest.mark.asyncio @@ -6055,7 +6032,7 @@ async def test_init_hashicorp_vault_config_override_retries_on_transport_error() assert reconnect_kwargs["reason"] == "init_hashicorp_vault_config_override_lookup_failure" -def test_update_config_fields_uppercases_env_vars(monkeypatch): +def test_settings_store_uppercases_db_env_vars(monkeypatch): """ Ensure environment variables pulled from DB are uppercased when applied so integrations like Datadog that expect uppercase env keys can read them. @@ -6066,13 +6043,12 @@ def test_update_config_fields_uppercases_env_vars(monkeypatch): monkeypatch.delenv(key, raising=False) proxy_config = ProxyConfig() - updated_config = proxy_config._update_config_fields( - current_config={}, - param_name="environment_variables", - db_param_value={"dd_api_key": "test-api-key", "dd_site": "us5.datadoghq.com"}, + db_values = proxy_config._prepared_db_settings_values( + "environment_variables", {"dd_api_key": "test-api-key", "dd_site": "us5.datadoghq.com"} ) + proxy_config.environment_variables.apply_db_row("environment_variables", db_values) - env_vars = updated_config.get("environment_variables", {}) + env_vars = proxy_config.environment_variables.resolved() assert env_vars["DD_API_KEY"] == "test-api-key" assert env_vars["DD_SITE"] == "us5.datadoghq.com" assert os.environ.get("DD_API_KEY") == "test-api-key" @@ -6529,9 +6505,8 @@ def test_get_config_normalizes_string_callbacks(monkeypatch): def test_deep_merge_dicts_skips_none_and_empty_lists(monkeypatch): - """ - Test that _update_config_fields deep merge skips None values and empty lists. - """ + """A key the config file declares is config-owned, so the stored row cannot + reshape it. Keys the file omits still come from the row.""" from litellm.proxy.proxy_server import ProxyConfig proxy_config = ProxyConfig() @@ -6557,14 +6532,14 @@ def test_deep_merge_dicts_skips_none_and_empty_lists(monkeypatch): }, } - result = proxy_config._update_config_fields(current_config, "general_settings", db_param_value) + proxy_config.settings.load_yaml(current_config["general_settings"]) + proxy_config.settings.apply_db_row("general_settings", db_param_value) + result = proxy_config.settings.resolved() - assert result["general_settings"]["max_parallel_requests"] == 10 - assert result["general_settings"]["allowed_models"] == ["gpt-3.5-turbo", "gpt-4"] - assert result["general_settings"]["new_key"] == "new_value" - assert result["general_settings"]["nested"]["key1"] == "updated_value1" - assert result["general_settings"]["nested"]["key2"] == "value2" - assert result["general_settings"]["nested"]["key3"] == "value3" + assert result["max_parallel_requests"] == 10 + assert result["allowed_models"] == ["gpt-3.5-turbo", "gpt-4"] + assert result["new_key"] == "new_value" + assert result["nested"] == {"key1": "value1", "key2": "value2"} class TestInvitationEndpoints: @@ -7408,17 +7383,20 @@ async def test_update_general_settings_clears_a_spend_log_cleanup_bound_dropped_ proxy_config = ProxyConfig() - with patch( - "litellm.proxy.proxy_server.general_settings", - {"maximum_spend_logs_cleanup_run_budget": "90s", "maximum_spend_logs_cleanup_batch_timeout": "10s"}, - ): + with patch("litellm.proxy.proxy_server.general_settings", proxy_config.settings): + await proxy_config._update_general_settings( + db_general_settings={ + "maximum_spend_logs_cleanup_run_budget": "90s", + "maximum_spend_logs_cleanup_batch_timeout": "10s", + } + ) await proxy_config._update_general_settings( db_general_settings={"maximum_spend_logs_cleanup_batch_timeout": "10s"} ) import litellm.proxy.proxy_server as ps - assert ps.general_settings["maximum_spend_logs_cleanup_run_budget"] is None + assert "maximum_spend_logs_cleanup_run_budget" not in ps.general_settings assert ps.general_settings["maximum_spend_logs_cleanup_batch_timeout"] == "10s" @@ -7429,9 +7407,9 @@ async def test_update_general_settings_keeps_a_yaml_set_spend_log_cleanup_bound( from litellm.proxy.proxy_server import ProxyConfig proxy_config = ProxyConfig() - proxy_config._yaml_spend_log_cleanup_bounds = {"maximum_spend_logs_cleanup_run_budget": "90s"} + proxy_config.settings.load_yaml({"maximum_spend_logs_cleanup_run_budget": "90s"}) - with patch("litellm.proxy.proxy_server.general_settings", {"maximum_spend_logs_cleanup_run_budget": "90s"}): + with patch("litellm.proxy.proxy_server.general_settings", proxy_config.settings): await proxy_config._update_general_settings(db_general_settings={"store_model_in_db": True}) import litellm.proxy.proxy_server as ps @@ -7447,10 +7425,10 @@ async def test_update_general_settings_clearing_a_db_override_falls_back_to_the_ from litellm.proxy.proxy_server import ProxyConfig proxy_config = ProxyConfig() - proxy_config._yaml_spend_log_cleanup_bounds = {"maximum_spend_logs_cleanup_run_budget": "90s"} + proxy_config.settings.load_yaml({"maximum_spend_logs_cleanup_run_budget": "90s"}) - # Memory currently holds the dashboard override, and the DB no longer carries it. - with patch("litellm.proxy.proxy_server.general_settings", {"maximum_spend_logs_cleanup_run_budget": "30s"}): + with patch("litellm.proxy.proxy_server.general_settings", proxy_config.settings): + await proxy_config._update_general_settings(db_general_settings={"maximum_spend_logs_cleanup_run_budget": "30s"}) await proxy_config._update_general_settings(db_general_settings={"store_model_in_db": True}) import litellm.proxy.proxy_server as ps @@ -7464,9 +7442,9 @@ async def test_update_general_settings_apply_user_budget_to_team_keys_yaml_wins( from litellm.proxy.proxy_server import ProxyConfig proxy_config = ProxyConfig() - proxy_config._yaml_general_settings_keys = {"apply_user_budget_to_team_keys"} + proxy_config.settings.load_yaml({"apply_user_budget_to_team_keys": True}) - with patch("litellm.proxy.proxy_server.general_settings", {"apply_user_budget_to_team_keys": True}): + with patch("litellm.proxy.proxy_server.general_settings", proxy_config.settings): await proxy_config._update_general_settings(db_general_settings={"apply_user_budget_to_team_keys": False}) import litellm.proxy.proxy_server as ps @@ -7514,14 +7492,13 @@ async def test_update_general_settings_keeps_yaml_pass_through_endpoints_next_to [(None, None), (["POST"], ["GET"])], ids=["all-methods", "disjoint-methods"], ) -async def test_update_general_settings_db_pass_through_endpoint_overrides_yaml_entry_on_the_same_path( +async def test_update_general_settings_db_pass_through_endpoint_cannot_override_a_yaml_declared_path( db_methods: list[str] | None, yaml_methods: list[str] | None ): - """The auth check matches pass-through entries by path only and lets any - matching ``auth: false`` entry through, so a DB ``auth: true`` entry can only - lock down a YAML-declared path if the YAML entry is dropped from the merged - list, whatever ``methods`` either entry declares.""" - from litellm.proxy._types import ProxyException + """``pass_through_endpoints`` is config-owned once the file declares it, so a stored + ``auth: true`` entry on a path the YAML already declares ``auth: false`` no longer + locks that path down. Changing it means editing the config file. A path the YAML + does not declare is still governed by the stored row, which the sibling test covers.""" from litellm.proxy.proxy_server import ProxyConfig yaml_endpoint: Final = { @@ -7551,9 +7528,71 @@ async def test_update_general_settings_db_pass_through_endpoint_overrides_yaml_e with settings, yaml_endpoints, initialize, master_key: await ProxyConfig()._update_general_settings(db_general_settings={"pass_through_endpoints": [db_endpoint]}) - with pytest.raises(ProxyException) as locked_down: - await user_api_key_auth(request=request, api_key=None) - assert locked_down.value.code == "401" + still_open: Final = await user_api_key_auth(request=request, api_key=None) + assert still_open.api_key is None + + +@pytest.mark.asyncio +async def test_deleting_the_stored_pass_through_row_takes_the_route_out_of_service(): + """A pass-through route the database declared has to stop serving when that row is + deleted. The proxy's own registry of live pass-through routes is what decides whether + a request is routed upstream or falls through to the auth error, so it has to lose the + entry on the reload rather than at the next process restart.""" + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import InitPassThroughEndpointHelpers + from litellm.proxy.proxy_server import ProxyConfig + + path: Final = f"/v1/deleted-{uuid.uuid4().hex[:8]}" + db_endpoint: Final = {"id": "db-1", "path": path, "target": "https://example.com/post"} + + def live_routes() -> set[str]: + return {route for route in InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() if path in route} + + settings: Final = patch("litellm.proxy.proxy_server.general_settings", {}) # test-quality-ok: the method reads this module global; no injection seam + yaml_endpoints: Final = patch("litellm.proxy.proxy_server.config_passthrough_endpoints", None) # test-quality-ok: module global holding the YAML endpoints; this case has none + with settings, yaml_endpoints: + pc = ProxyConfig() + await pc._update_general_settings(db_general_settings={"pass_through_endpoints": [db_endpoint]}) + assert live_routes(), "the stored endpoint should be serving before the row is deleted" + + await pc._update_general_settings(db_general_settings={}) + + assert live_routes() == set() + + +@pytest.mark.asyncio +async def test_a_stored_pass_through_row_never_disturbs_the_config_declared_routes(): + """``pass_through_endpoints`` is config-owned once the file declares it, so writing and then + deleting a stored row resolves to the same list both times and the config file's routes keep + serving untouched. The stored entry never gets a route of its own.""" + from litellm.proxy.pass_through_endpoints.pass_through_endpoints import ( + InitPassThroughEndpointHelpers, + initialize_pass_through_endpoints, + ) + from litellm.proxy.proxy_server import ProxyConfig + + marker: Final = uuid.uuid4().hex[:8] + config_path: Final = f"/v1/kept-{marker}" + db_path: Final = f"/v1/ignored-{marker}" + config_endpoint: Final = {"id": f"cfg-{marker}", "path": config_path, "target": "https://example.com/post"} + db_endpoint: Final = {"id": f"db-{marker}", "path": db_path, "target": "https://example.com/post"} + + def live_paths() -> set[str]: + registered: Final = InitPassThroughEndpointHelpers.get_all_registered_pass_through_routes() + return {path for path in (config_path, db_path) if any(path in route for route in registered)} + + settings: Final = patch("litellm.proxy.proxy_server.general_settings", {"pass_through_endpoints": [config_endpoint]}) # test-quality-ok: the method reads this module global; no injection seam + yaml_endpoints: Final = patch("litellm.proxy.proxy_server.config_passthrough_endpoints", [config_endpoint]) # test-quality-ok: module global holding the YAML endpoints the reload merges in + with settings, yaml_endpoints: + await initialize_pass_through_endpoints(pass_through_endpoints=[config_endpoint]) + assert live_paths() == {config_path} + + pc = ProxyConfig() + await pc._update_general_settings(db_general_settings={"pass_through_endpoints": [db_endpoint]}) + assert live_paths() == {config_path} + + await pc._update_general_settings(db_general_settings={}) + + assert live_paths() == {config_path} def _fill_user_api_key_cache(cache: DualCache, count: int) -> None: @@ -7587,10 +7626,11 @@ async def test_update_general_settings_clearing_user_api_key_cache_max_size_rest from litellm.proxy.proxy_server import ProxyConfig cache = UserApiKeyCache() - cache.update_in_memory_max_size(5000) - monkeypatch.setattr(proxy_server_module, "general_settings", {"user_api_key_cache_max_size": 5000}) + proxy_config = ProxyConfig() + monkeypatch.setattr(proxy_server_module, "general_settings", proxy_config.settings) monkeypatch.setattr(proxy_server_module, "user_api_key_cache", cache) - await ProxyConfig()._update_general_settings(db_general_settings={"store_model_in_db": True}) + await proxy_config._update_general_settings(db_general_settings={"user_api_key_cache_max_size": 5000}) + await proxy_config._update_general_settings(db_general_settings={"store_model_in_db": True}) assert "user_api_key_cache_max_size" not in proxy_server_module.general_settings @@ -7625,10 +7665,10 @@ async def test_update_general_settings_user_api_key_cache_max_size_yaml_wins(mon from litellm.proxy.proxy_server import ProxyConfig proxy_config = ProxyConfig() - proxy_config._yaml_general_settings_keys = {"user_api_key_cache_max_size"} + proxy_config.settings.load_yaml({"user_api_key_cache_max_size": 300}) cache = UserApiKeyCache() cache.update_in_memory_max_size(300) - monkeypatch.setattr(proxy_server_module, "general_settings", {"user_api_key_cache_max_size": 300}) + monkeypatch.setattr(proxy_server_module, "general_settings", proxy_config.settings) monkeypatch.setattr(proxy_server_module, "user_api_key_cache", cache) await proxy_config._update_general_settings(db_general_settings={"user_api_key_cache_max_size": 10}) @@ -7661,7 +7701,10 @@ async def test_update_general_settings_disable_auto_add_proxy_admin_to_teams(db_ import litellm.proxy.proxy_server as ps - assert ps.general_settings["disable_auto_add_proxy_admin_to_teams"] is expected + if expected is None: + assert "disable_auto_add_proxy_admin_to_teams" not in ps.general_settings + else: + assert ps.general_settings["disable_auto_add_proxy_admin_to_teams"] is expected @pytest.mark.asyncio @@ -11175,11 +11218,8 @@ def test_prompt_caching_settings_propagate_on_config_reload(monkeypatch, field_n monkeypatch.setattr(litellm, field_name, False if isinstance(db_value, bool) else None) pc = ps.ProxyConfig() - pc._update_config_fields( - current_config={"litellm_settings": {}}, - param_name="litellm_settings", - db_param_value={field_name: db_value}, - ) + resolved_db_values = pc._prepared_db_settings_values("litellm_settings", {field_name: db_value}) + pc._apply_litellm_settings_db_values(resolved_db_values) assert getattr(litellm, field_name) == db_value @@ -11453,6 +11493,7 @@ def _config_field_info_client(monkeypatch, user_role): from fastapi.testclient import TestClient import litellm.proxy.proxy_server as ps + from litellm.proxy.config_resolvers import SettingsStore from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.proxy_server import app @@ -11475,6 +11516,12 @@ def _config_field_info_client(monkeypatch, user_role): mock_prisma = MagicMock() mock_prisma.db = types.SimpleNamespace(litellm_config=mock_config_table) monkeypatch.setattr(ps, "prisma_client", mock_prisma) + + settings = SettingsStore("general_settings") + settings.load_yaml({}) + settings.apply_db_row("general_settings", db_record.param_value) + monkeypatch.setattr(ps.proxy_config, "settings", settings) + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth(user_id="u", user_role=user_role) return TestClient(app) @@ -11670,6 +11717,217 @@ async def test_update_config_general_settings_emits_audit_log(monkeypatch): assert before["some_api_key"] != "sk-stored-secret" +@pytest.mark.asyncio +async def test_delete_config_general_settings_is_visible_to_the_next_read(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import ConfigFieldDelete + from litellm.proxy.config_resolvers import SettingsStore + from litellm.proxy.proxy_server import delete_config_general_settings, get_config_general_settings + + fake = _fake_prisma_with_config({"max_request_size_mb": 42}) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + + settings = SettingsStore("general_settings") + settings.load_yaml({}) + settings.apply_db_row("general_settings", {"max_request_size_mb": 42}) + monkeypatch.setattr(proxy_server_module.proxy_config, "settings", settings) + + admin = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + await delete_config_general_settings( + data=ConfigFieldDelete(field_name="max_request_size_mb", config_type="general_settings"), + user_api_key_dict=admin, + ) + + with pytest.raises(HTTPException) as excinfo: + await get_config_general_settings(field_name="max_request_size_mb", user_api_key_dict=admin) + assert excinfo.value.status_code == 400 + assert "is not set" in excinfo.value.detail["error"] + + +@pytest.mark.asyncio +async def test_ui_litellm_field_write_refuses_a_key_the_config_file_declares(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import ConfigFieldUpdate + from litellm.proxy.proxy_server import ProxyConfig, update_config_general_settings + + pc = ProxyConfig() + pc._load_yaml_settings_stores({"litellm_settings": {"enable_anthropic_prompt_caching": True}}) + monkeypatch.setattr(proxy_server_module, "proxy_config", pc) + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + + fake = _fake_prisma_with_config({}) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + + admin = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + with pytest.raises(HTTPException) as excinfo: + await update_config_general_settings( + data=ConfigFieldUpdate( + field_name="enable_anthropic_prompt_caching", field_value=False, config_type="general_settings" + ), + user_api_key_dict=admin, + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.detail["keys"] == ["enable_anthropic_prompt_caching"] + assert litellm.enable_anthropic_prompt_caching is True + fake.db.litellm_config.upsert.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_ui_litellm_field_reset_refuses_a_key_the_config_file_declares(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy.proxy_server import ProxyConfig, _reset_general_settings_ui_litellm_field + + pc = ProxyConfig() + pc._load_yaml_settings_stores({"litellm_settings": {"enable_anthropic_prompt_caching": True}}) + monkeypatch.setattr(proxy_server_module, "proxy_config", pc) + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + + fake = _fake_prisma_with_config({}) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + + admin = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + with pytest.raises(HTTPException) as excinfo: + await _reset_general_settings_ui_litellm_field("enable_anthropic_prompt_caching", admin) + + assert excinfo.value.status_code == 400 + assert litellm.enable_anthropic_prompt_caching is True + + +@pytest.mark.asyncio +async def test_update_config_general_settings_refuses_a_key_the_config_file_declares(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import ConfigFieldUpdate + from litellm.proxy.proxy_server import ProxyConfig, update_config_general_settings + + pc = ProxyConfig() + pc._load_yaml_settings_stores({"general_settings": {"max_parallel_requests": 111}}) + monkeypatch.setattr(proxy_server_module, "proxy_config", pc) + monkeypatch.setattr(proxy_server_module, "user_config_file_path", "/etc/litellm/config.yaml") + + fake = _fake_prisma_with_config({}) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + + admin = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + with pytest.raises(HTTPException) as excinfo: + await update_config_general_settings( + data=ConfigFieldUpdate( + field_name="max_parallel_requests", field_value=999, config_type="general_settings" + ), + user_api_key_dict=admin, + ) + + assert excinfo.value.status_code == 400 + detail = excinfo.value.detail + assert detail["keys"] == ["max_parallel_requests"] + assert "max_parallel_requests" in detail["error"] + assert "/etc/litellm/config.yaml" in detail["resolution"] + fake.db.litellm_config.upsert.assert_not_awaited() + assert pc.settings["max_parallel_requests"] == 111 + + +@pytest.mark.asyncio +async def test_save_config_refuses_a_key_the_config_file_declares(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy.proxy_server import ProxyConfig + + pc = ProxyConfig() + pc._load_yaml_settings_stores({"general_settings": {"max_parallel_requests": 111}}) + monkeypatch.setattr(proxy_server_module, "proxy_config", pc) + + fake = _fake_prisma_with_config({}) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + + with pytest.raises(HTTPException) as excinfo: + await pc._save_changed_config_section( + section_name="general_settings", + baseline={"general_settings": {"max_parallel_requests": 111}}, + new_config={"general_settings": {"max_parallel_requests": 999}}, + prisma_client=fake, + ) + + assert excinfo.value.status_code == 400 + assert excinfo.value.detail["keys"] == ["max_parallel_requests"] + fake.db.litellm_config.upsert.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_save_config_allows_a_write_that_matches_the_config_file(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy.proxy_server import ProxyConfig + + pc = ProxyConfig() + pc._load_yaml_settings_stores({"general_settings": {"max_parallel_requests": 111}}) + monkeypatch.setattr(proxy_server_module, "proxy_config", pc) + + fake = _fake_prisma_with_config({}) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + + await pc._save_changed_config_section( + section_name="general_settings", + baseline={"general_settings": {}}, + new_config={"general_settings": {"max_parallel_requests": 111, "max_request_size_mb": 42}}, + prisma_client=fake, + ) + + assert pc.settings["max_request_size_mb"] == 42 + assert pc.settings["max_parallel_requests"] == 111 + + +@pytest.mark.asyncio +async def test_update_config_general_settings_is_visible_to_the_next_read(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy._types import ConfigFieldUpdate + from litellm.proxy.config_resolvers import SettingsStore + from litellm.proxy.proxy_server import ( + get_config_general_settings, + update_config_general_settings, + ) + + fake = _fake_prisma_with_config({}) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + + settings = SettingsStore("general_settings") + settings.load_yaml({}) + monkeypatch.setattr(proxy_server_module.proxy_config, "settings", settings) + + admin = UserAPIKeyAuth(api_key="hashed-admin", user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + await update_config_general_settings( + data=ConfigFieldUpdate(field_name="max_request_size_mb", field_value=42, config_type="general_settings"), + user_api_key_dict=admin, + ) + + read_back = await get_config_general_settings(field_name="max_request_size_mb", user_api_key_dict=admin) + assert read_back.field_value == 42 + assert read_back.source == "db" + assert read_back.editable is True + + +@pytest.mark.asyncio +async def test_save_config_makes_a_db_owned_write_visible_to_the_next_read(monkeypatch): + import litellm.proxy.proxy_server as proxy_server_module + from litellm.proxy.proxy_server import ProxyConfig + + pc = ProxyConfig() + pc._load_yaml_settings_stores({"general_settings": {"max_parallel_requests": 111}}) + monkeypatch.setattr(proxy_server_module, "proxy_config", pc) + + fake = _fake_prisma_with_config({}) + monkeypatch.setattr(proxy_server_module, "prisma_client", fake) + + await pc._save_changed_config_section( + section_name="general_settings", + baseline={"general_settings": {}}, + new_config={"general_settings": {"max_request_size_mb": 42}}, + prisma_client=fake, + ) + + assert pc.settings["max_request_size_mb"] == 42 + assert pc.settings.source("max_request_size_mb") == "db" + assert pc.settings["max_parallel_requests"] == 111 + assert pc.settings.source("max_parallel_requests") == "config" + + @pytest.mark.asyncio async def test_update_config_field_rejects_out_of_range_alerting_args(monkeypatch): """Out-of-range alerting_args must be rejected at save time. If they land in the @@ -13893,8 +14151,8 @@ def test_disabling_docs_does_not_disable_other_routes(monkeypatch): "db_general_settings, expected", [ ({"enable_openai_websocket_passthrough": True}, True), - ({"enable_openai_websocket_passthrough": False}, False), - ({}, None), + ({"enable_openai_websocket_passthrough": False}, True), + ({}, True), ], ) async def test_update_general_settings_propagates_openai_websocket_passthrough(db_general_settings, expected): @@ -13915,9 +14173,9 @@ async def test_update_general_settings_keeps_yaml_openai_websocket_passthrough() from litellm.proxy.proxy_server import ProxyConfig proxy_config = ProxyConfig() - proxy_config._yaml_general_settings_keys = {"enable_openai_websocket_passthrough"} + proxy_config.settings.load_yaml({"enable_openai_websocket_passthrough": False}) - with patch("litellm.proxy.proxy_server.general_settings", {"enable_openai_websocket_passthrough": False}): + with patch("litellm.proxy.proxy_server.general_settings", proxy_config.settings): await proxy_config._update_general_settings(db_general_settings={"enable_openai_websocket_passthrough": True}) import litellm.proxy.proxy_server as ps diff --git a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py index 8f17a1e45de..fc5733b1a82 100644 --- a/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py +++ b/tests/test_litellm/proxy/ui_crud_endpoints/test_proxy_setting_endpoints.py @@ -3439,3 +3439,31 @@ class TestSyncUiSettingsToGeneralSettings: assert dict(applied) == {} assert general_settings == {"allow_agents_for_team_admins": True} + + def test_applied_runtime_flags_keep_the_ui_row_as_the_source(self, monkeypatch): + from litellm.proxy import proxy_server + from litellm.proxy.config_resolvers import SettingsStore + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import apply_runtime_general_settings_flags + + general_settings = SettingsStore("general_settings") + general_settings.load_yaml({}) + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + + apply_runtime_general_settings_flags({"forward_client_headers_to_llm_api": True}) + + assert general_settings["forward_client_headers_to_llm_api"] is True + assert general_settings.source("forward_client_headers_to_llm_api") == "db" + + def test_applied_runtime_flags_cannot_override_the_config_file(self, monkeypatch): + from litellm.proxy import proxy_server + from litellm.proxy.config_resolvers import SettingsStore + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import apply_runtime_general_settings_flags + + general_settings = SettingsStore("general_settings") + general_settings.load_yaml({"forward_client_headers_to_llm_api": False}) + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + + apply_runtime_general_settings_flags({"forward_client_headers_to_llm_api": True}) + + assert general_settings["forward_client_headers_to_llm_api"] is False + assert general_settings.source("forward_client_headers_to_llm_api") == "config" diff --git a/tests/test_litellm/repositories/test_repositories.py b/tests/test_litellm/repositories/test_repositories.py index c7f6b8ff83a..b6b4a8072fa 100644 --- a/tests/test_litellm/repositories/test_repositories.py +++ b/tests/test_litellm/repositories/test_repositories.py @@ -1447,27 +1447,6 @@ class TestConfigRepository: client = MockPrismaClient() return ConfigRepository(client) - def test_deep_merge_dicts_db_wins(self, repo): - dst = {"a": 1, "b": {"c": 2}} - src = {"a": 10, "b": {"d": 3}} - repo._deep_merge_dicts(dst, src) - assert dst["a"] == 10 - assert dst["b"]["c"] == 2 - assert dst["b"]["d"] == 3 - - def test_deep_merge_dicts_skips_none(self, repo): - dst = {"a": 1} - src = {"a": None, "b": 2} - repo._deep_merge_dicts(dst, src) - assert dst["a"] == 1 - assert dst["b"] == 2 - - def test_deep_merge_dicts_skips_empty_list(self, repo): - dst = {"models": ["gpt-4"]} - src = {"models": []} - repo._deep_merge_dicts(dst, src) - assert dst["models"] == ["gpt-4"] - @pytest.mark.asyncio async def test_get_param(self, repo): repo._prisma_client.db.litellm_config._records["general_settings"] = { @@ -1512,99 +1491,6 @@ class TestConfigRepository: params = await repo.get_all_params() assert len(params) == 2 - @pytest.mark.asyncio - async def test_reconcile_config_skips_when_store_model_false(self, repo): - yaml_config = {"general_settings": {"key": "value"}} - result = await repo.reconcile_config(yaml_config, store_model_in_db=False) - assert result == yaml_config - - @pytest.mark.asyncio - async def test_prefetch_params(self, repo): - repo._prisma_client.db.litellm_config._records["general_settings"] = { - "param_name": "general_settings", - "param_value": "{}", - } - await repo.prefetch_params(["general_settings"]) - - @pytest.mark.asyncio - async def test_reconcile_config_with_db_values(self, repo): - repo._prisma_client.db.litellm_config._records["general_settings"] = { - "param_name": "general_settings", - "param_value": '{"master_key": "db-key", "db_only": "from_db"}', - } - repo._prisma_client.db.litellm_config._records["router_settings"] = { - "param_name": "router_settings", - "param_value": '{"timeout": 60}', - } - yaml_config = { - "general_settings": {"master_key": "yaml-key", "yaml_only": "from_yaml"}, - } - result = await repo.reconcile_config(yaml_config, store_model_in_db=True) - assert result["general_settings"]["master_key"] == "db-key" - assert result["general_settings"]["yaml_only"] == "from_yaml" - assert result["general_settings"]["db_only"] == "from_db" - assert result["router_settings"]["timeout"] == 60 - - @pytest.mark.asyncio - @patch("litellm.repositories.config_repository.decrypt_value_helper") - async def test_reconcile_config_with_environment_variables( - self, mock_decrypt, repo - ): - mock_decrypt.side_effect = lambda value, **kw: f"decrypted_{value}" - repo._prisma_client.db.litellm_config._records["environment_variables"] = { - "param_name": "environment_variables", - "param_value": '{"api_key": "encrypted_key", "secret": "encrypted_secret"}', - } - yaml_config = {} - result = await repo.reconcile_config(yaml_config, store_model_in_db=True) - assert "environment_variables" in result - assert "api_key" in result["environment_variables"] - assert "API_KEY" in result["environment_variables"] - - @pytest.mark.asyncio - async def test_reconcile_config_none_values_preserved(self, repo): - repo._prisma_client.db.litellm_config._records["general_settings"] = { - "param_name": "general_settings", - "param_value": '{"new_key": "value", "null_key": null}', - } - yaml_config = {"general_settings": {"existing": "keep"}} - result = await repo.reconcile_config(yaml_config, store_model_in_db=True) - assert result["general_settings"]["existing"] == "keep" - assert result["general_settings"]["new_key"] == "value" - - def test_update_config_fields_non_dict(self, repo): - config = {"litellm_settings": "old_value"} - result = repo._update_config_fields( - current_config=config, - param_name="litellm_settings", - db_param_value="new_value", - ) - assert result["litellm_settings"] == "new_value" - - def test_update_config_fields_new_param(self, repo): - config = {} - result = repo._update_config_fields( - current_config=config, - param_name="router_settings", - db_param_value={"timeout": 30}, - ) - assert result["router_settings"] == {"timeout": 30} - - @patch("litellm.repositories.config_repository.decrypt_value_helper") - def test_decrypt_env_variables_non_string(self, mock_decrypt, repo): - mock_decrypt.side_effect = lambda value, **kw: value - env_vars = {"string_val": "encrypted", "int_val": 123, "bool_val": True} - result = repo._decrypt_env_variables(env_vars) - assert result["int_val"] == "123" - assert result["bool_val"] == "True" - - @patch("litellm.repositories.config_repository.decrypt_value_helper") - def test_decrypt_env_variables_none_value(self, mock_decrypt, repo): - mock_decrypt.return_value = None - env_vars = {"key": "value"} - result = repo._decrypt_env_variables(env_vars) - assert "key" not in result - class TestVerificationTokenRepositoryExtended: @pytest.fixture @@ -2213,48 +2099,6 @@ class TestTeamRepositoryArchiveData: assert "router_settings" in archive_data -class TestConfigRepositoryDeepCopy: - @pytest.fixture - def repo(self): - client = MockPrismaClient() - return ConfigRepository(client) - - @pytest.mark.asyncio - async def test_reconcile_config_does_not_mutate_original(self, repo): - import copy - - repo._prisma_client.db.litellm_config._records["general_settings"] = { - "param_name": "general_settings", - "param_value": '{"db_key": "db_value", "nested": {"db_nested": "from_db"}}', - } - original_config = { - "general_settings": { - "yaml_key": "yaml_value", - "nested": {"yaml_nested": "from_yaml"}, - } - } - original_copy = copy.deepcopy(original_config) - result = await repo.reconcile_config(original_config, store_model_in_db=True) - assert original_config == original_copy - assert result["general_settings"]["db_key"] == "db_value" - assert result["general_settings"]["yaml_key"] == "yaml_value" - assert result["general_settings"]["nested"]["db_nested"] == "from_db" - assert result["general_settings"]["nested"]["yaml_nested"] == "from_yaml" - - @pytest.mark.asyncio - async def test_reconcile_config_repeated_calls_independent(self, repo): - repo._prisma_client.db.litellm_config._records["general_settings"] = { - "param_name": "general_settings", - "param_value": '{"db_key": "db_value"}', - } - yaml_config = {"general_settings": {"yaml_key": "yaml_value"}} - result1 = await repo.reconcile_config(yaml_config, store_model_in_db=True) - result1["general_settings"]["modified"] = "in_result1" - result2 = await repo.reconcile_config(yaml_config, store_model_in_db=True) - assert "modified" not in yaml_config.get("general_settings", {}) - assert "modified" not in result2.get("general_settings", {}) - - class TestPrismaTableRepository: def test_table_property_returns_named_delegate(self): from litellm.proxy.common_utils.config_sync_pubsub import ( diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 51a7a196d0a..1a49bb6ddc1 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -437,16 +437,6 @@ "count": 1 } }, - "src/app/(dashboard)/hooks/projects/useDeleteProject.test.ts": { - "react/display-name": { - "count": 1 - } - }, - "src/app/(dashboard)/hooks/projects/useDeleteProject.ts": { - "no-restricted-syntax": { - "count": 1 - } - }, "src/app/(dashboard)/hooks/projects/useProjectDetails.test.ts": { "react/display-name": { "count": 1 @@ -1066,7 +1056,7 @@ "count": 1 }, "prefer-const": { - "count": 2 + "count": 1 } }, "src/app/(dashboard)/search-tools/_components/SearchTools.tsx": { @@ -1685,9 +1675,6 @@ "src/components/key_team_helpers/key_list.tsx": { "local/filename-pascal-case": { "count": 1 - }, - "react-hooks/set-state-in-effect": { - "count": 1 } }, "src/components/key_team_helpers/transform_key_info.tsx": { @@ -1804,16 +1791,16 @@ "count": 1 }, "max-params": { - "count": 23 + "count": 21 }, "no-nested-ternary": { "count": 5 }, "no-restricted-syntax": { - "count": 150 + "count": 147 }, "prefer-const": { - "count": 32 + "count": 31 } }, "src/components/object_permissions_view.tsx": { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/types.ts b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/types.ts index 2414e5b8f31..a9f9f7375ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/types.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/access-groups/_components/types.ts @@ -12,35 +12,3 @@ export interface AccessGroup { updatedAt: string; updatedBy: string; } - -export interface Model { - id: string; - name: string; - provider: string; -} - -export interface McpServer { - id: string; - name: string; - endpoint: string; -} - -export interface Agent { - id: string; - name: string; - type: string; -} - -export interface AccessGroupKey { - id: string; - alias: string; - status: string; - createdAt: string; -} - -export interface AccessGroupTeam { - id: string; - name: string; - members: number; - role: string; -} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.integration.test.tsx index d4fbb7e153e..3de88fb6f57 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.integration.test.tsx @@ -40,7 +40,7 @@ vi.mock("@/components/llm_calls/fetch_models", () => ({ fetchAvailableModels: vi.fn().mockResolvedValue([]), })); -vi.mock("@/components/HelpLink", () => ({ +vi.mock("@/components/DocsMenu", () => ({ DocsMenu: () => null, })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.test.tsx index a8609aef629..50c8c12c9c2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.test.tsx @@ -41,7 +41,7 @@ vi.mock("@/components/llm_calls/fetch_models", () => ({ fetchAvailableModels: vi.fn().mockResolvedValue([]), })); -vi.mock("@/components/HelpLink", () => ({ +vi.mock("@/components/DocsMenu", () => ({ DocsMenu: () => null, })); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx index 52b66edcbe5..b8e51939dd0 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/cost_tracking_settings.tsx @@ -19,7 +19,7 @@ import AddProviderForm from "./add_provider_form"; import ProviderMarginTable from "./provider_margin_table"; import AddMarginForm from "./add_margin_form"; import PricingCalculator from "./pricing_calculator/index"; -import { DocsMenu } from "@/components/HelpLink"; +import { DocsMenu } from "@/components/DocsMenu"; import HowItWorks from "./how_it_works"; import { useDiscountConfig } from "./use_discount_config"; import { useMarginConfig } from "./use_margin_config"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/index.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/index.ts index 90701dd8f1f..4771000a2e2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/index.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/index.ts @@ -1,16 +1 @@ export { default as CostTrackingSettings } from "./cost_tracking_settings"; -export { default as ProviderDiscountTable } from "./provider_discount_table"; -export { default as AddProviderForm } from "./add_provider_form"; -export { default as ProviderMarginTable } from "./provider_margin_table"; -export { default as AddMarginForm } from "./add_margin_form"; -export { default as HowItWorks } from "./how_it_works"; -export type { - CostTrackingSettingsProps, - DiscountConfig, - CostDiscountResponse, - MarginConfig, - CostMarginResponse, -} from "./types"; -export * from "./provider_display_helpers"; -export { useDiscountConfig } from "./use_discount_config"; -export { useMarginConfig } from "./use_margin_config"; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/types.ts b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/types.ts index f824e2f1eff..07a807df66e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/types.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/cost-tracking/_components/types.ts @@ -8,18 +8,10 @@ export interface DiscountConfig { [provider: string]: number; } -export interface CostDiscountResponse { - values: DiscountConfig; -} - export interface MarginConfig { [provider: string]: number | { percentage?: number; fixed_amount?: number }; } -export interface CostMarginResponse { - values: MarginConfig; -} - export interface CostEstimateRequest { model: string; input_tokens: number; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.test.tsx deleted file mode 100644 index 60bf235040f..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.test.tsx +++ /dev/null @@ -1,88 +0,0 @@ -import { render, screen, act } from "@testing-library/react"; -import userEvent from "@testing-library/user-event"; -import { vi } from "vitest"; -import { GuardrailConfig } from "./GuardrailConfig"; - -describe("GuardrailConfig", () => { - const defaultProps = { - guardrailName: "Content Safety", - guardrailType: "Content Safety", - provider: "bedrock", - }; - - afterEach(() => { - vi.useRealTimers(); - }); - - it("should render", () => { - render(); - expect(screen.getByText("Parameters")).toBeInTheDocument(); - }); - - it("should display the guardrail name in the parameters description", () => { - render(); - expect(screen.getByText(/Configure Content Safety behavior/)).toBeInTheDocument(); - }); - - // Note: Version history entries are hardcoded placeholders in the component. - // These assertions will need updating when wired to real API data. - it("should show version history when 'View history' is clicked", async () => { - const user = userEvent.setup(); - render(); - await user.click(screen.getByRole("button", { name: /view history/i })); - expect(screen.getByText("Initial configuration")).toBeInTheDocument(); - expect(screen.getByText("Added custom categories list")).toBeInTheDocument(); - }); - - it("should toggle version history text between View/Hide", async () => { - const user = userEvent.setup(); - render(); - const button = screen.getByRole("button", { name: /view history/i }); - await user.click(button); - expect(screen.getByRole("button", { name: /hide history/i })).toBeInTheDocument(); - }); - - it("should show custom code textarea when custom code override is toggled on", async () => { - const user = userEvent.setup(); - render(); - await user.click(screen.getByRole("switch", { name: "Custom Code Override" })); - expect(screen.getByPlaceholderText(/async def evaluate/)).toBeInTheDocument(); - }); - - it("should hide custom code textarea when custom code override is off", () => { - render(); - // There's an input for categories, but no textarea - expect(screen.queryByPlaceholderText(/async def evaluate/)).not.toBeInTheDocument(); - }); - - it("should show the re-run button in idle state", () => { - render(); - expect(screen.getByRole("button", { name: /re-run on failing logs/i })).toBeInTheDocument(); - }); - - it("should show loading state when re-run is clicked", async () => { - vi.useFakeTimers({ shouldAdvanceTime: true }); - const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); - render(); - await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); - expect(screen.getByText(/Running on 10 samples/)).toBeInTheDocument(); - }); - - it("should show success message after re-run completes", async () => { - vi.useFakeTimers({ shouldAdvanceTime: true }); - const user = userEvent.setup({ advanceTimers: vi.advanceTimersByTime }); - render(); - await user.click(screen.getByRole("button", { name: /re-run on failing logs/i })); - await act(async () => { - vi.advanceTimersByTime(2500); - }); - expect(screen.getByText(/7\/10 would now pass/)).toBeInTheDocument(); - }); - - it("should display the Revert and Save buttons", () => { - render(); - expect(screen.getByRole("button", { name: /revert/i })).toBeInTheDocument(); - // The component's hardcoded default version is "v3", so Save shows "v4" - expect(screen.getByRole("button", { name: /save as v\d+/i })).toBeInTheDocument(); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx deleted file mode 100644 index 34da9b8d08d..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails-monitor/_components/GuardrailConfig.tsx +++ /dev/null @@ -1,261 +0,0 @@ -import { CircleCheck, CirclePlay, Code, Save, Undo2 } from "lucide-react"; -import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; -import { Button } from "@/components/ui/button"; -import { Input } from "@/components/ui/input"; -import { Label } from "@/components/ui/label"; -import { Switch } from "@/components/ui/switch"; -import { Textarea } from "@/components/ui/textarea"; -import React, { useId, useState } from "react"; - -interface GuardrailConfigProps { - guardrailName: string; - guardrailType: string; - provider: string; -} - -const versions = [ - { - id: "v3", - label: "v3 (current)", - date: "2026-02-18", - author: "admin@company.com", - changes: "Adjusted sensitivity for medical terms", - }, - { id: "v2", label: "v2", date: "2026-02-10", author: "admin@company.com", changes: "Added custom categories list" }, - { id: "v1", label: "v1", date: "2026-01-28", author: "admin@company.com", changes: "Initial configuration" }, -]; - -const ACTION_ITEMS = [ - { value: "block", label: "Block Request" }, - { value: "flag", label: "Flag for Review" }, - { value: "log", label: "Log Only" }, - { value: "fallback", label: "Use Fallback Response" }, -]; - -const PROVIDER_ITEMS = [ - { value: "bedrock", label: "AWS Bedrock Guardrails" }, - { value: "google", label: "Google Cloud AI Safety" }, - { value: "litellm", label: "LiteLLM Built-in" }, - { value: "custom", label: "Custom Code" }, -]; - -const GUARDRAIL_TYPE_ITEMS = [ - { value: "Content Safety", label: "Content Safety" }, - { value: "PII", label: "PII Detection" }, - { value: "Topic", label: "Topic Restriction" }, - { value: "prompt_injection", label: "Prompt Injection" }, - { value: "custom", label: "Custom" }, -]; - -export function GuardrailConfig({ guardrailName, guardrailType, provider }: GuardrailConfigProps) { - const [action, setAction] = useState("block"); - const [enabled, setEnabled] = useState(true); - const [customCode, setCustomCode] = useState(""); - const [useCustomCode, setUseCustomCode] = useState(false); - const [rerunStatus, setRerunStatus] = useState<"idle" | "running" | "success" | "error">("idle"); - const [version, setVersion] = useState("v3"); - const [showVersionHistory, setShowVersionHistory] = useState(false); - const enabledToggleId = useId(); - - const handleRerun = () => { - setRerunStatus("running"); - setTimeout(() => { - setRerunStatus("success"); - setTimeout(() => setRerunStatus("idle"), 3000); - }, 2000); - }; - - return ( -
- {/* Version Bar */} -
-
-
- Version: - - -
-
- - -
-
- - {showVersionHistory && ( -
- {versions.map((v) => ( -
-
- - {v.id} - - {v.changes} -
-
- {v.author} - {v.date} -
-
- ))} -
- )} -
- - {/* Parameters */} -
-

Parameters

-

Configure {guardrailName} behavior

- -
-
- - -
- -
- - -
- -
- - -
- -
- - -
- -
- - -
-
-
- - {/* Custom Code Override */} -
-
-
-

- - Custom Code Override -

-

- Replace the built-in guardrail with custom evaluation code -

-
- -
- - {useCustomCode && ( -