chore(mcp): merge main with unit test timeout safeguards

.circleci/config.yml

@@ -3015,7 +3015,7 @@ workflows:
           name: integration-<< matrix.suite >>
           matrix:
             parameters:
-              suite: [management, accounting, database, providers, extensions, sdk, browser]
+              suite: [management, accounting, database, providers, extensions, sdk, cost, browser]
           filters:
             branches:
               only:

.circleci/scripts/run_integration.sh

@@ -9,6 +9,7 @@ fi
 suite="${1:?integration suite required}"
 results="test-results/integration-${suite}"
 mkdir -p "$results"
+shard_timeout=11m
 integration_identity="$(.venv/bin/python -c 'import uuid; print(uuid.uuid4().hex)')"
 upstream_pid=""
 proxy_pid=""
@@ -108,13 +109,26 @@ awk '$3 == "REJECT" && $1 > 0 { rejected=1 } END { exit !rejected }' "$results/e
 setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
   .venv/bin/python -m integration._support.upstream > "$results/upstream.log" 2>&1 &
 upstream_pid=$!
+if [ "$suite" = cost ]; then
+  export INTEGRATION_WORKERS=8
+fi
 start_proxy() {
   local port="$1"
   local log_name="$2"
+  local -a cost_map_env
+  if [ "$suite" = cost ]; then
+    cost_map_env=(
+      "LITELLM_MODEL_COST_MAP_URL=$INTEGRATION_UPSTREAM_URL/_cost_map"
+      "MODEL_COST_MAP_MIN_MODEL_COUNT=1"
+      "MODEL_COST_MAP_MAX_SHRINK_RATIO=0"
+    )
+  else
+    cost_map_env=("LITELLM_LOCAL_MODEL_COST_MAP=True")
+  fi
   setsid env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" INTEGRATION_RUN_ID="$integration_identity" \
     DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
     LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" LITELLM_SALT_KEY="$LITELLM_SALT_KEY" LITELLM_UI_PATH="$LITELLM_UI_PATH" PROXY_BASE_URL="http://127.0.0.1:$port" \
-    LITELLM_MODE=PRODUCTION LITELLM_LOCAL_MODEL_COST_MAP=True STORE_MODEL_IN_DB=True \
+    LITELLM_MODE=PRODUCTION STORE_MODEL_IN_DB=True "${cost_map_env[@]}" \
     AWS_EC2_METADATA_DISABLED=true DO_NOT_TRACK=1 \
     .venv/bin/python -m integration._support.proxy --config tests/integration/proxy_config.yaml \
     --host 127.0.0.1 --port "$port" --num_workers 1 --telemetry False \
@@ -158,11 +172,12 @@ if [ "$suite" = browser ]; then
   exit 0
 fi
 
-timeout --signal=TERM --kill-after=20s 11m env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
+timeout --signal=TERM --kill-after=20s "$shard_timeout" env -i PATH="$PATH" HOME="$HOME" PYTHONPATH="$PYTHONPATH" \
   INTEGRATION_RUN_ID="$integration_identity" \
   DATABASE_URL="$DATABASE_URL" REDIS_HOST="$REDIS_HOST" REDIS_PORT="$REDIS_PORT" \
   INTEGRATION_PROXY_URL="$INTEGRATION_PROXY_URL" INTEGRATION_PEER_URL="$INTEGRATION_PEER_URL" \
   INTEGRATION_UPSTREAM_URL="$INTEGRATION_UPSTREAM_URL" \
+  INTEGRATION_WORKERS="${INTEGRATION_WORKERS:-1}" \
   INTEGRATION_MASTER_KEY="$INTEGRATION_MASTER_KEY" LITELLM_MODE=PRODUCTION \
   INTEGRATION_SEED="$INTEGRATION_SEED" \
   INTEGRATION_ORDER_SEED="$INTEGRATION_ORDER_SEED" \

.github/workflows/_test-unit-base.yml

@@ -37,6 +37,18 @@ on:
         required: false
         type: number
         default: 60
+      test-timeout-seconds:
+        description: >-
+          Per-test ceiling enforced by pytest-timeout, covering fixture setup and
+          teardown as well as the test body. A test that hangs fails with a
+          traceback of where it was stuck instead of idling the shard until
+          `timeout-minutes` cancels it. Timed-out tests are excluded from reruns
+          because pytest-timeout arms its timer once per test and
+          pytest-rerunfailures reruns inside that same window, so a rerun of a
+          timed-out test would run with no timer at all.
+        required: false
+        type: number
+        default: 120
       max-failures:
         description: "Stop after this many failures"
         required: false
@@ -137,6 +149,7 @@ jobs:
           MAX_FAILURES: ${{ inputs.max-failures }}
           WORKERS: ${{ inputs.workers }}
           RERUNS: ${{ inputs.reruns }}
+          TEST_TIMEOUT_SECONDS: ${{ inputs.test-timeout-seconds }}
           DIST: ${{ inputs.dist }}
           COVERAGE_CORE: sysmon
         run: |
@@ -146,6 +159,8 @@ jobs:
               --maxfail="${MAX_FAILURES}" \
               --reruns "${RERUNS}" \
               --reruns-delay 1 \
+              --timeout="${TEST_TIMEOUT_SECONDS}" \
+              --rerun-except "from pytest-timeout" \
               --durations=20 \
               --cov=./litellm --cov=./enterprise/litellm_enterprise \
               --cov-report=xml:coverage.xml \
@@ -157,6 +172,8 @@ jobs:
               -n "${WORKERS}" \
               --reruns "${RERUNS}" \
               --reruns-delay 1 \
+              --timeout="${TEST_TIMEOUT_SECONDS}" \
+              --rerun-except "from pytest-timeout" \
               --dist="${DIST}" \
               --durations=20 \
               --cov=./litellm --cov=./enterprise/litellm_enterprise \

litellm-rust/Cargo.lock

@@ -1976,6 +1976,7 @@ dependencies = [
  "azure_identity",
  "litellm-auth",
  "moka",
+ "rstest",
  "serde_json",
  "sha2 0.10.9",
  "strum",
@@ -2137,11 +2138,14 @@ version = "0.1.0"
 dependencies = [
  "http 1.4.2",
  "hyper-util",
+ "litellm-core-utils",
  "reqwest 0.12.28",
  "rstest",
  "rustls 0.23.42",
+ "serde_json",
  "thiserror 2.0.19",
  "tokio",
+ "veil",
  "webpki-roots",
 ]
 
@@ -2187,6 +2191,7 @@ dependencies = [
  "litellm-auth-gcp",
  "litellm-callbacks-legacy",
  "litellm-core",
+ "litellm-core-utils",
  "litellm-host-python",
  "litellm-http",
  "litellm-llms",

litellm-rust/crates/auth-azure/Cargo.toml

@@ -18,4 +18,5 @@ azure_core = "1.0.0"
 azure_identity = { version = "1.0.0", features = ["tokio"] }
 
 [dev-dependencies]
+rstest.workspace = true
 tokio.workspace = true

litellm-rust/crates/auth-azure/src/types.rs

@@ -1,11 +1,10 @@
-use serde_json::{Map, Value};
 use std::collections::BTreeMap;
-use strum::EnumString;
 
-use litellm_auth::Error;
 use litellm_auth::{
-    CredentialResolverHandle, InputSource, SecretValue, Sourced, TokenProviderHandle,
+    CredentialResolverHandle, Error, InputSource, SecretValue, Sourced, TokenProviderHandle,
 };
+use serde_json::{Map, Value};
+use strum::EnumString;
 
 pub const DEFAULT_AZURE_SCOPE: &str = "https://cognitiveservices.azure.com/.default";
 
@@ -52,6 +51,16 @@ pub struct AzureAuthInputs {
 }
 
 impl AzureAuthInputs {
+    pub fn or_configured_token_refresh(self, enabled: bool) -> Self {
+        if *self.enable_azure_ad_token_refresh.value() || !enabled {
+            return self;
+        }
+        Self {
+            enable_azure_ad_token_refresh: Sourced::new(true, InputSource::Deployment),
+            ..self
+        }
+    }
+
     #[cfg(test)]
     pub fn from_optional_params(params: &Map<String, Value>) -> Result<Self, Error> {
         Self::from_sourced_optional_params(params, &BTreeMap::new())
@@ -115,12 +124,12 @@ fn source_for(sources: &BTreeMap<String, InputSource>, name: &str) -> InputSourc
 
 #[cfg(test)]
 mod tests {
-    use serde_json::json;
-
     use std::collections::BTreeMap;
 
-    use super::{AzureAuthInputs, AzureCredentialType, ConfigValue};
     use litellm_auth::{InputSource, Sourced};
+    use serde_json::json;
+
+    use super::{AzureAuthInputs, AzureCredentialType, ConfigValue};
 
     #[test]
     fn selector_parsing_is_exact() {
@@ -189,4 +198,29 @@ mod tests {
         assert!(!debug.contains("token-value"));
         assert!(!debug.contains("secret-value"));
     }
+
+    #[rstest::rstest]
+    #[case::global_turns_refresh_on(json!({}), true, true, InputSource::Deployment)]
+    #[case::global_overrides_a_call_false_like_python(json!({"enable_azure_ad_token_refresh": false}), true, true, InputSource::Deployment)]
+    #[case::call_true_survives_a_global_false(json!({"enable_azure_ad_token_refresh": true}), false, true, InputSource::Request)]
+    #[case::both_off(json!({}), false, false, InputSource::Request)]
+    fn token_refresh_follows_the_configured_global(
+        #[case] params: serde_json::Value,
+        #[case] global: bool,
+        #[case] enabled: bool,
+        #[case] source: InputSource,
+    ) {
+        let sources = BTreeMap::from([(
+            "enable_azure_ad_token_refresh".to_string(),
+            InputSource::Request,
+        )]);
+        let inputs =
+            AzureAuthInputs::from_sourced_optional_params(params.as_object().unwrap(), &sources)
+                .unwrap()
+                .or_configured_token_refresh(global);
+        assert_eq!(
+            inputs.enable_azure_ad_token_refresh,
+            Sourced::new(enabled, source)
+        );
+    }
 }

litellm-rust/crates/auth-gcp/src/lib.rs

@@ -1,17 +1,13 @@
-use std::collections::BTreeMap;
-use std::future::Future;
-use std::path::Path;
-use std::pin::Pin;
-use std::sync::Arc;
+use std::{collections::BTreeMap, future::Future, path::Path, pin::Pin, sync::Arc};
 
 use gcp_auth::{CustomServiceAccount, TokenProvider};
+use litellm_auth::{
+    CredentialPlacement, Error, InputSource, SecretValue, Sourced, http::apply_credential,
+};
 use moka::future::Cache;
 use serde_json::{Map, Value};
 use sha2::{Digest, Sha256};
 
-use litellm_auth::http::apply_credential;
-use litellm_auth::{CredentialPlacement, Error, InputSource, SecretValue, Sourced};
-
 const CLOUD_PLATFORM_SCOPE: &str = "https://www.googleapis.com/auth/cloud-platform";
 const GOOGLE_OAUTH_TOKEN_ENDPOINT: &str = "https://oauth2.googleapis.com/token";
 const GOOGLE_APPLICATION_CREDENTIALS_ENV: &str = "GOOGLE_APPLICATION_CREDENTIALS";
@@ -45,6 +41,16 @@ impl VertexConfig {
         })
     }
 
+    pub fn or_configured(self, project_id: Option<&str>, location: Option<&str>) -> Self {
+        let configured =
+            |value: Option<&str>| value.filter(|value| !value.is_empty()).map(str::to_string);
+        Self {
+            project_id: self.project_id.or_else(|| configured(project_id)),
+            location: self.location.or_else(|| configured(location)),
+            ..self
+        }
+    }
+
     pub fn project_id(&self) -> Option<&str> {
         self.project_id.as_deref()
     }
@@ -571,4 +577,29 @@ mod tests {
         assert_eq!(loads.load(Ordering::SeqCst), 1);
         assert_eq!(calls.load(Ordering::SeqCst), 4);
     }
+
+    #[test]
+    fn configured_defaults_sit_between_call_params_and_the_environment() {
+        let env = |name: &str| Some(format!("env-{name}"));
+        let from_config =
+            VertexConfig::default().or_configured(Some("global-project"), Some("global-location"));
+        assert_eq!(
+            get_vertex_ai_project(&from_config, &env).as_deref(),
+            Some("global-project")
+        );
+        assert_eq!(
+            get_vertex_ai_location(&from_config, &env).as_deref(),
+            Some("global-location")
+        );
+        let from_call =
+            config(json!({"vertex_project":"call-project","vertex_location":"call-location"}))
+                .or_configured(Some("global-project"), Some("global-location"));
+        assert_eq!(from_call.project_id(), Some("call-project"));
+        assert_eq!(from_call.location(), Some("call-location"));
+        let empty_global = VertexConfig::default().or_configured(Some(""), None);
+        assert_eq!(
+            get_vertex_ai_project(&empty_global, &env).as_deref(),
+            Some("env-VERTEXAI_PROJECT")
+        );
+    }
 }

litellm-rust/crates/core-utils/src/lib.rs

@@ -6,4 +6,5 @@ pub mod params;
 pub mod prompt_templates;
 pub mod secret_redaction;
 pub mod serde_compat;
+pub mod settings;
 pub mod url_utils;

litellm-rust/crates/core-utils/src/settings.rs

@@ -0,0 +1,144 @@
+use std::str::FromStr;
+
+pub trait Lookup {
+    fn get(&self, name: &str) -> Option<String>;
+
+    fn truthy(&self, name: &str) -> Option<String> {
+        self.get(name).filter(|value| !value.is_empty())
+    }
+
+    fn enabled(&self, name: &str) -> Option<bool> {
+        self.get(name)
+            .is_some_and(|value| value.trim().eq_ignore_ascii_case("true"))
+            .then_some(true)
+    }
+
+    fn parsed<T: FromStr>(&self, name: &str) -> Option<T>
+    where
+        Self: Sized,
+    {
+        self.get(name).and_then(|value| value.trim().parse().ok())
+    }
+}
+
+impl<F: Fn(&str) -> Option<String>> Lookup for F {
+    fn get(&self, name: &str) -> Option<String> {
+        self(name)
+    }
+}
+
+pub struct ProcessEnvironment;
+
+impl Lookup for ProcessEnvironment {
+    fn get(&self, name: &str) -> Option<String> {
+        std::env::var(name).ok()
+    }
+}
+
+pub trait Layer: Default {
+    fn or(self, lower: Self) -> Self;
+}
+
+pub fn merge<L: Layer>(highest_precedence_first: impl IntoIterator<Item = L>) -> L {
+    highest_precedence_first
+        .into_iter()
+        .reduce(L::or)
+        .unwrap_or_default()
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
+        move |name| {
+            values
+                .iter()
+                .find(|(key, _)| *key == name)
+                .map(|(_, value)| value.to_string())
+        }
+    }
+
+    #[test]
+    fn get_keeps_a_present_empty_value_like_os_getenv_with_a_fallback() {
+        let env = env_of(&[("EMPTY", "")]);
+        assert_eq!(env.get("EMPTY"), Some(String::new()));
+        assert_eq!(env.get("ABSENT"), None);
+    }
+
+    #[test]
+    fn truthy_drops_an_empty_value_like_a_python_or_chain() {
+        let env = env_of(&[("EMPTY", ""), ("SET", "value")]);
+        assert_eq!(env.truthy("EMPTY"), None);
+        assert_eq!(env.truthy("SET").as_deref(), Some("value"));
+    }
+
+    #[test]
+    fn enabled_only_switches_on_for_true_and_never_forces_off() {
+        let env = env_of(&[
+            ("LOWER", "true"),
+            ("PADDED", " True "),
+            ("OFF", "false"),
+            ("ONE", "1"),
+        ]);
+        assert_eq!(env.enabled("LOWER"), Some(true));
+        assert_eq!(env.enabled("PADDED"), Some(true));
+        assert_eq!(env.enabled("OFF"), None);
+        assert_eq!(env.enabled("ONE"), None);
+        assert_eq!(env.enabled("ABSENT"), None);
+    }
+
+    #[test]
+    fn parsed_trims_and_skips_values_that_do_not_parse() {
+        let env = env_of(&[("PADDED", " 45 "), ("WORD", "soon"), ("FRACTION", "0.5")]);
+        assert_eq!(env.parsed::<u32>("PADDED"), Some(45));
+        assert_eq!(env.parsed::<u32>("WORD"), None);
+        assert_eq!(env.parsed::<f64>("FRACTION"), Some(0.5));
+        assert_eq!(env.parsed::<u32>("ABSENT"), None);
+    }
+
+    #[derive(Debug, Default, PartialEq)]
+    struct Pair {
+        first: Option<u8>,
+        second: Option<u8>,
+    }
+
+    impl Layer for Pair {
+        fn or(self, lower: Self) -> Self {
+            Self {
+                first: self.first.or(lower.first),
+                second: self.second.or(lower.second),
+            }
+        }
+    }
+
+    #[test]
+    fn merge_takes_each_field_from_the_highest_layer_that_sets_it() {
+        let merged = merge([
+            Pair {
+                first: Some(1),
+                second: None,
+            },
+            Pair {
+                first: Some(2),
+                second: Some(2),
+            },
+            Pair {
+                first: Some(3),
+                second: Some(3),
+            },
+        ]);
+        assert_eq!(
+            merged,
+            Pair {
+                first: Some(1),
+                second: Some(2),
+            }
+        );
+    }
+
+    #[test]
+    fn merging_no_layers_yields_the_empty_layer() {
+        assert_eq!(merge(Vec::<Pair>::new()), Pair::default());
+    }
+}

litellm-rust/crates/core/AGENTS.md

@@ -5,10 +5,11 @@ litellm-core is the LiteLLM SDK in Rust — it makes the LLM call. Each top-leve
 Each crate mirrors one top-level Python package, so a Rust path reads as its Python path with the crate name in place of the package directory. Dependencies only point down:
 
 - `litellm-types` mirrors `litellm/types/`: pure serde data, no I/O
-- `litellm-core-utils` mirrors `litellm/litellm_core_utils/`: pure helpers (provider resolution, prompt factory, call arguments), no network I/O
-- `litellm-llms` mirrors `litellm/llms/`: `base_llm/<api>/transformation.rs`, `<provider>/<api>/transformation.rs`, and `custom_httpx/` (HTTP helpers and the OCR request handler)
+- `litellm-core-utils` mirrors `litellm/litellm_core_utils/`: pure helpers (provider resolution, prompt factory, call arguments, settings lookup and layer merge), no network I/O
+- `litellm-http` is Rust-only and route-neutral: settings resolution, the pooled `reqwest` clients, TLS, proxies, the SSRF-safe media fetcher, request and header helpers, and transport errors. Python's `litellm/llms/custom_httpx/` is split by responsibility instead of mirrored: its transport half lives here, its OCR handler in `litellm-llms`
+- `litellm-llms` mirrors `litellm/llms/`: `base_llm/<api>/transformation.rs`, `<provider>/<api>/transformation.rs`, and `base_llm/ocr/handler.rs` (the OCR request handler)
 - `litellm-core` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...): entrypoints, route request types, provider dispatch, the route machine, and hooks
 
-A route module owns the call entrypoint, route request types (`*Request<'a>`), credential fallback, provider dispatch, and the handler glue that runs a provider config. Provider code never imports from core; when it needs the caller's hooks mid-call it goes through `litellm_llms::custom_httpx::llm_http_handler::CallHooks`, which each route implements over its host. Import every item from its canonical path. Never re-export another crate's items or give an item a second public path; the only re-export allowed is a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`). Handlers belong in core or llms, never in a host crate
+A route module owns the call entrypoint, route request types (`*Request<'a>`), credential fallback, provider dispatch, and the handler glue that runs a provider config. Provider code never imports from core; when it needs the caller's hooks mid-call it goes through `litellm_llms::base_llm::ocr::handler::CallHooks`, which each route implements over its host. Import every item from its canonical path. Never re-export another crate's items or give an item a second public path; the only re-export allowed is a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`). Handlers belong in core or llms, never in a host crate
 
 Not here: serving HTTP (axum routes, extractors), config file reading, rollout state, databases, or callback execution of any kind. Core runs each route as a machine that yields host operations and call events; which integrations consume those events is the host's business.

litellm-rust/crates/core/Cargo.toml

@@ -15,6 +15,7 @@ futures-util.workspace = true
 base64.workspace = true
 litellm-auth.workspace = true
 litellm-auth-aws.workspace = true
+litellm-http.workspace = true
 litellm-llms.workspace = true
 moka.workspace = true
 mime_guess = "2.0.5"
@@ -36,7 +37,6 @@ veil.workspace = true
 
 [dev-dependencies]
 litellm-auth-gcp.workspace = true
-litellm-http.workspace = true
 litellm-llms = { workspace = true, features = ["test-support"] }
 rstest.workspace = true
 rstest_reuse.workspace = true

litellm-rust/crates/core/src/audio_transcription/error.rs

@@ -20,9 +20,9 @@ pub enum Error {
     #[error(transparent)]
     Auth(#[from] litellm_auth::Error),
     #[error(transparent)]
-    Transport(#[from] litellm_llms::custom_httpx::transport::Error),
+    Transport(#[from] litellm_http::transport::Error),
     #[error(transparent)]
-    Headers(#[from] litellm_llms::custom_httpx::http_handler::HeaderError),
+    Headers(#[from] litellm_http::request::HeaderError),
     #[error(transparent)]
     Aws(#[from] litellm_auth_aws::Error),
 }

litellm-rust/crates/core/src/audio_transcription/handler.rs

@@ -1,4 +1,4 @@
-use litellm_llms::custom_httpx::http_handler::{http_request, truncate_error_body};
+use litellm_http::request::{http_request, truncate_error_body};
 use serde_json::Value;
 
 use super::{Error, client::http_client};
@@ -18,23 +18,17 @@ pub async fn execute_audio_transcription_provider_call(
         request_builder = request_builder.timeout(duration);
     }
     let response = http_request(request_builder).await.map_err(|error| {
-        Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
-            error.to_string(),
-        ))
+        Error::Transport(litellm_http::transport::Error::Network(error.to_string()))
     })?;
     let status = response.status();
     let text = response.text().await.map_err(|error| {
-        Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
-            error.to_string(),
-        ))
+        Error::Transport(litellm_http::transport::Error::Network(error.to_string()))
     })?;
     if !status.is_success() {
-        return Err(Error::Transport(
-            litellm_llms::custom_httpx::transport::Error::Http {
-                status: status.as_u16(),
-                body: truncate_error_body(&text),
-            },
-        ));
+        return Err(Error::Transport(litellm_http::transport::Error::Http {
+            status: status.as_u16(),
+            body: truncate_error_body(&text),
+        }));
     }
     let response_json = serde_json::from_str(&text)
         .map_err(|error| Error::InvalidResponse(format!("invalid audio response JSON: {error}")))?;

litellm-rust/crates/core/src/audio_transcription/prepare.rs

@@ -1,10 +1,10 @@
 use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
+use litellm_http::request::{has_header, string_headers};
 use litellm_llms::{
     base_llm::audio_transcription::transformation::{
         AudioTranscriptionAuth, BaseAudioTranscriptionConfig,
     },
     bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
-    custom_httpx::http_handler::{has_header, string_headers},
 };
 
 use super::Error;

litellm-rust/crates/core/src/chat_completions/common_utils.rs

@@ -1,8 +1,8 @@
+use litellm_http::request::string_headers as shared_string_headers;
 use litellm_llms::{
     anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG,
     base_llm::chat::transformation::BaseConfig,
     bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
-    custom_httpx::http_handler::string_headers as shared_string_headers,
 };
 use serde_json::{Map, Value};
 

litellm-rust/crates/core/src/chat_completions/error.rs

@@ -20,9 +20,9 @@ pub enum Error {
     #[error(transparent)]
     Auth(#[from] litellm_auth::Error),
     #[error(transparent)]
-    Transport(#[from] litellm_llms::custom_httpx::transport::Error),
+    Transport(#[from] litellm_http::transport::Error),
     #[error(transparent)]
-    Headers(#[from] litellm_llms::custom_httpx::http_handler::HeaderError),
+    Headers(#[from] litellm_http::request::HeaderError),
     #[error(transparent)]
     Aws(#[from] litellm_auth_aws::Error),
 }

litellm-rust/crates/core/src/chat_completions/handler.rs

@@ -1,7 +1,5 @@
-use litellm_llms::{
-    base_llm::chat::transformation::{ChatCompletionsAuth, ProviderChatResponseData},
-    custom_httpx::http_handler::{http_request, truncate_error_body},
-};
+use litellm_http::request::{http_request, truncate_error_body};
+use litellm_llms::base_llm::chat::transformation::{ChatCompletionsAuth, ProviderChatResponseData};
 use litellm_types::utils::ChatCompletionsResponse;
 use serde_json::Value;
 
@@ -34,30 +32,22 @@ pub(super) async fn execute_chat_completions_provider_call(
         // so the host can still serve it. Everything else here, a timeout
         // above all, may have reached the provider and been answered.
         if err.is_connect() || err.is_builder() {
-            Error::Transport(litellm_llms::custom_httpx::transport::Error::Connect(
-                err.to_string(),
-            ))
+            Error::Transport(litellm_http::transport::Error::Connect(err.to_string()))
         } else {
-            Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
-                err.to_string(),
-            ))
+            Error::Transport(litellm_http::transport::Error::Network(err.to_string()))
         }
     })?;
 
     let status = response.status();
     let text = response.text().await.map_err(|err| {
-        Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
-            err.to_string(),
-        ))
+        Error::Transport(litellm_http::transport::Error::Network(err.to_string()))
     })?;
 
     if !status.is_success() {
-        return Err(Error::Transport(
-            litellm_llms::custom_httpx::transport::Error::Http {
-                status: status.as_u16(),
-                body: truncate_error_body(&text),
-            },
-        ));
+        return Err(Error::Transport(litellm_http::transport::Error::Http {
+            status: status.as_u16(),
+            body: truncate_error_body(&text),
+        }));
     }
 
     let body: Value = serde_json::from_str(&text).map_err(|err| {
@@ -82,9 +72,7 @@ pub(super) async fn execute_chat_completions_provider_call(
 pub(super) fn as_response_error(err: Error) -> Error {
     match err {
         already @ (Error::InvalidResponse(_)
-        | Error::Transport(litellm_llms::custom_httpx::transport::Error::Http {
-            ..
-        })) => already,
+        | Error::Transport(litellm_http::transport::Error::Http { .. })) => already,
         other => Error::InvalidResponse(other.to_string()),
     }
 }

litellm-rust/crates/core/src/chat_completions/prepare.rs

@@ -1,8 +1,6 @@
 use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
-use litellm_llms::{
-    base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth},
-    custom_httpx::http_handler::has_header,
-};
+use litellm_http::request::has_header;
+use litellm_llms::base_llm::chat::transformation::{BaseConfig, ChatCompletionsAuth};
 use litellm_types::llms::openai::ChatMessage;
 use serde_json::Value;
 

litellm-rust/crates/core/src/chat_completions/tests.rs

@@ -265,7 +265,7 @@ fn rejects_non_string_extra_headers() {
     call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))]));
     assert_eq!(
         decline(call),
-        Error::Headers(litellm_llms::custom_httpx::http_handler::HeaderError {
+        Error::Headers(litellm_http::request::HeaderError {
             context: "chat completions",
             name: "x-trace".to_string(),
             actual: "number",
@@ -771,10 +771,7 @@ mod round_trip {
         assert!(
             matches!(
                 err,
-                Error::Transport(litellm_llms::custom_httpx::transport::Error::Http {
-                    status: 429,
-                    ..
-                })
+                Error::Transport(litellm_http::transport::Error::Http { status: 429, .. })
             ),
             "expected a 429, got {err:?}"
         );
@@ -801,7 +798,7 @@ mod round_trip {
         assert!(
             matches!(
                 err,
-                Error::Transport(litellm_llms::custom_httpx::transport::Error::Connect(_))
+                Error::Transport(litellm_http::transport::Error::Connect(_))
             ),
             "expected a pre-send connect failure, got {err:?}"
         );
@@ -825,16 +822,11 @@ mod round_trip {
         }
         // An upstream status is already unambiguous, so it survives intact.
         assert!(matches!(
-            as_response_error(Error::Transport(
-                litellm_llms::custom_httpx::transport::Error::Http {
-                    status: 500,
-                    body: "boom".to_string()
-                }
-            )),
-            Error::Transport(litellm_llms::custom_httpx::transport::Error::Http {
+            as_response_error(Error::Transport(litellm_http::transport::Error::Http {
                 status: 500,
-                ..
-            })
+                body: "boom".to_string()
+            })),
+            Error::Transport(litellm_http::transport::Error::Http { status: 500, .. })
         ));
     }
 }

litellm-rust/crates/core/src/messages/common_utils.rs

@@ -1,11 +1,9 @@
-pub(super) use litellm_llms::custom_httpx::http_handler::{
-    has_bearer_auth, has_header, truncate_error_body,
-};
+use litellm_http::request::string_headers as shared_string_headers;
+pub(super) use litellm_http::request::{has_bearer_auth, has_header, truncate_error_body};
 use litellm_llms::{
     anthropic::experimental_pass_through::messages::transformation::ANTHROPIC_MESSAGES_CONFIG,
     azure_ai::anthropic::messages_transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG,
     base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig,
-    custom_httpx::http_handler::string_headers as shared_string_headers,
 };
 use serde_json::{Map, Value};
 

litellm-rust/crates/core/src/messages/error.rs

@@ -15,9 +15,9 @@ pub enum Error {
     #[error(transparent)]
     Auth(#[from] litellm_auth::Error),
     #[error(transparent)]
-    Transport(#[from] litellm_llms::custom_httpx::transport::Error),
+    Transport(#[from] litellm_http::transport::Error),
     #[error(transparent)]
-    Headers(#[from] litellm_llms::custom_httpx::http_handler::HeaderError),
+    Headers(#[from] litellm_http::request::HeaderError),
 }
 
 impl From<LlmError> for Error {

litellm-rust/crates/core/src/messages/handler.rs

@@ -1,9 +1,7 @@
 use std::time::Duration;
 
-use litellm_llms::{
-    base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig,
-    custom_httpx::{http_handler::http_request, transport::Error as TransportError},
-};
+use litellm_http::{request::http_request, transport::Error as TransportError};
+use litellm_llms::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig;
 use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse;
 use serde_json::Value;
 

litellm-rust/crates/core/src/messages/tests.rs

@@ -82,7 +82,7 @@ fn string_headers_rejects_non_string_values() {
     let err = string_headers(Some(headers)).expect_err("non-string header rejected");
     assert_eq!(
         err,
-        Error::Headers(litellm_llms::custom_httpx::http_handler::HeaderError {
+        Error::Headers(litellm_http::request::HeaderError {
             context: "messages",
             name: "x-count".to_string(),
             actual: "number",
@@ -432,7 +432,7 @@ async fn messages_maps_provider_error_status_to_http_error() {
 
     assert!(matches!(
         err,
-        Error::Transport(litellm_llms::custom_httpx::transport::Error::Http { status: 401, .. })
+        Error::Transport(litellm_http::transport::Error::Http { status: 401, .. })
     ));
 }
 

litellm-rust/crates/core/src/ocr/client.rs

@@ -1,6 +1,5 @@
-use litellm_llms::{
-    base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse},
-    custom_httpx::llm_http_handler::OcrClient,
+use litellm_llms::base_llm::ocr::{
+    error::Error, handler::OcrClient, transformation::LiteLLMOcrResponse,
 };
 
 use crate::ocr::{

litellm-rust/crates/core/src/ocr/handler.rs

@@ -1,12 +1,10 @@
 use futures_util::future::BoxFuture;
 use litellm_auth::SecretValue;
 use litellm_host::event::{MachineEvent, RawResponse, RequestContext, WireRequest};
-use litellm_llms::{
-    base_llm::ocr::{
-        error::Error,
-        transformation::{LiteLLMOcrResponse, PreparedOcrRequest},
-    },
-    custom_httpx::llm_http_handler::{CallHooks, OcrClient},
+use litellm_llms::base_llm::ocr::{
+    error::Error,
+    handler::{CallHooks, OcrClient},
+    transformation::{LiteLLMOcrResponse, PreparedOcrRequest},
 };
 use serde_json::Value;
 
@@ -24,7 +22,7 @@ pub(crate) async fn perform_ocr_request(
 ) -> Result<LiteLLMOcrResponse, Error> {
     request.response_format()?;
     let config = request.config;
-    let request = prepare_request(request, caller_document);
+    let request = prepare_request(request, caller_document, client);
     let hooks = OcrCallHooks::new(host.clone(), &request, config);
     config.ocr(client, &request, &hooks).await
 }

litellm-rust/crates/core/src/ocr/prepare.rs

@@ -1,6 +1,7 @@
 use litellm_auth::{InputSource, SecretValue, Sourced};
-use litellm_llms::base_llm::ocr::transformation::{
-    OcrConnection, OcrCredentialInputs, PreparedOcrRequest, credential_env,
+use litellm_llms::base_llm::ocr::{
+    handler::OcrClient,
+    transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest},
 };
 
 use super::provider_config::OcrProvider;
@@ -9,26 +10,31 @@ use crate::ocr::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
 pub(crate) fn prepare_request(
     request: ResolvedOcrRequest,
     caller_document: bool,
+    client: &OcrClient,
 ) -> PreparedOcrRequest {
     let credentials = request.credentials.clone();
-    let api_base_env = match request.config.provider() {
-        OcrProvider::Mistral => Some("MISTRAL_API_BASE"),
-        OcrProvider::AzureAi => Some("AZURE_AI_API_BASE"),
-        OcrProvider::Cohere | OcrProvider::Reducto | OcrProvider::VertexAi => None,
+    let (preferred_api_key_env, api_base_env) = match request.config.provider() {
+        OcrProvider::Mistral => (
+            Some("MISTRAL_AZURE_API_KEY"),
+            Some("MISTRAL_AZURE_API_BASE"),
+        ),
+        OcrProvider::AzureAi => (None, Some("AZURE_AI_API_BASE")),
+        OcrProvider::Cohere | OcrProvider::Reducto | OcrProvider::VertexAi => (None, None),
     };
+    let secret = |name: &str| client.secrets().truthy(name);
     let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
         credentials.api_key.clone().or_else(|| {
-            request
-                .config
-                .get_api_key_env_var()
-                .and_then(credential_env)
+            preferred_api_key_env
+                .into_iter()
+                .chain(request.config.get_api_key_env_var())
+                .find_map(secret)
                 .map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
         })
     });
     let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
         credentials.api_base.clone().or_else(|| {
             api_base_env
-                .and_then(credential_env)
+                .and_then(secret)
                 .map(|value| Sourced::new(value, InputSource::Environment))
         })
     });
@@ -51,7 +57,12 @@ pub(crate) fn prepare_request(
     PreparedOcrRequest {
         model,
         document,
-        connection: OcrConnection::new(resolved, transport),
+        connection: OcrConnection::new(
+            resolved,
+            transport,
+            client.settings().clone(),
+            client.secrets().clone(),
+        ),
         caller_document,
         optional_params,
         input_sources,
@@ -61,7 +72,11 @@ pub(crate) fn prepare_request(
 
 #[cfg(test)]
 pub(crate) fn prepare_request_for_test(request: ResolvedOcrRequest) -> PreparedOcrRequest {
-    prepare_request(request, true)
+    prepare_request(
+        request,
+        true,
+        &OcrClient::for_test(reqwest::Client::new(), reqwest::Client::new()),
+    )
 }
 
 #[cfg(test)]

litellm-rust/crates/core/src/ocr/provider_config.rs

@@ -7,13 +7,13 @@ use litellm_llms::{
     },
     base_llm::ocr::{
         error::Error,
+        handler::{self, CallHooks, OcrClient},
         transformation::{
             BaseOcrConfig, LiteLLMOcrResponse, OcrCredentialInputs, OcrDocument,
             PreparedOcrRequest, ResolvedOcrCredentials,
         },
     },
     cohere::ocr::transformation::CohereParseConfig,
-    custom_httpx::llm_http_handler::{self, CallHooks, OcrClient},
     mistral::ocr::transformation::MistralOcrConfig,
     reducto::ocr::transformation::{ReductoParseLegacyConfig, ReductoParseV3Config},
     vertex_ai::ocr::{
@@ -116,7 +116,7 @@ impl OcrConfigKind {
         request: &PreparedOcrRequest,
         hooks: &dyn CallHooks<Error>,
     ) -> Result<LiteLLMOcrResponse, Error> {
-        with_config!(self, config => llm_http_handler::ocr(&config, client, request, hooks).await)
+        with_config!(self, config => handler::ocr(&config, client, request, hooks).await)
     }
 }
 

litellm-rust/crates/core/src/ocr/route.rs

@@ -6,9 +6,8 @@ use litellm_host::{
     machine::{HostChannel, HostTokenProvider, MachineFault, RouteMachine, TokenRoute},
     route::Route,
 };
-use litellm_llms::{
-    base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse},
-    custom_httpx::llm_http_handler::OcrClient,
+use litellm_llms::base_llm::ocr::{
+    error::Error, handler::OcrClient, transformation::LiteLLMOcrResponse,
 };
 
 use super::handler::perform_ocr_request;

litellm-rust/crates/core/src/ocr/types.rs

@@ -277,7 +277,7 @@ mod tests {
             vec![("x-a".to_string(), "1".to_string())]
         );
         assert_eq!(request.transport.extra_headers_source, InputSource::Request);
-        assert_eq!(request.transport.timeout, Duration::from_secs(7));
+        assert_eq!(request.transport.timeout, Some(Duration::from_secs(7)));
         assert_eq!(request.input_sources.len(), 2);
 
         let defaulted = LiteLLMOcrRequest::from_inputs(

litellm-rust/crates/core/src/responses/error.rs

@@ -11,7 +11,7 @@ pub enum Error {
     #[error(transparent)]
     Auth(#[from] litellm_auth::Error),
     #[error(transparent)]
-    Transport(#[from] litellm_llms::custom_httpx::transport::Error),
+    Transport(#[from] litellm_http::transport::Error),
     #[error(transparent)]
-    Headers(#[from] litellm_llms::custom_httpx::http_handler::HeaderError),
+    Headers(#[from] litellm_http::request::HeaderError),
 }

litellm-rust/crates/core/src/responses/websocket.rs

@@ -95,9 +95,7 @@ impl ResponsesWebSocketConnection {
         timeout: Option<Duration>,
     ) -> Result<Self, Error> {
         let mut request = url.into_client_request().map_err(|error| {
-            Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
-                error.to_string(),
-            ))
+            Error::Transport(litellm_http::transport::Error::Network(error.to_string()))
         })?;
         for (name, value) in headers {
             let header_name = name
@@ -110,7 +108,7 @@ impl ResponsesWebSocketConnection {
         let connect = connect_upstream(request);
         let result = match timeout {
             Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| {
-                Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
+                Error::Transport(litellm_http::transport::Error::Network(
                     "Responses WebSocket connection timed out".into(),
                 ))
             })?,
@@ -118,14 +116,12 @@ impl ResponsesWebSocketConnection {
         };
         let (socket, _) = result.map_err(|error| match *error {
             tokio_tungstenite::tungstenite::Error::Http(response) => {
-                Error::Transport(litellm_llms::custom_httpx::transport::Error::Http {
+                Error::Transport(litellm_http::transport::Error::Http {
                     status: response.status().as_u16(),
                     body: String::new(),
                 })
             }
-            other => Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
-                other.to_string(),
-            )),
+            other => Error::Transport(litellm_http::transport::Error::Network(other.to_string())),
         })?;
         Ok(Self {
             socket: Arc::new(Mutex::new(Some(socket))),
@@ -135,16 +131,12 @@ impl ResponsesWebSocketConnection {
     pub async fn send_text(&self, text: String) -> Result<(), Error> {
         let mut socket = self.socket.lock().await;
         let Some(socket) = socket.as_mut() else {
-            return Err(Error::Transport(
-                litellm_llms::custom_httpx::transport::Error::Network(
-                    "Responses WebSocket is closed".into(),
-                ),
-            ));
+            return Err(Error::Transport(litellm_http::transport::Error::Network(
+                "Responses WebSocket is closed".into(),
+            )));
         };
         socket.send(Message::Text(text)).await.map_err(|error| {
-            Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
-                error.to_string(),
-            ))
+            Error::Transport(litellm_http::transport::Error::Network(error.to_string()))
         })
     }
 
@@ -160,9 +152,9 @@ impl ResponsesWebSocketConnection {
                 .map_err(|error| Error::InvalidResponse(error.to_string())),
             Some(Ok(Message::Close(_))) | None => Ok(None),
             Some(Ok(_)) => Ok(None),
-            Some(Err(error)) => Err(Error::Transport(
-                litellm_llms::custom_httpx::transport::Error::Network(error.to_string()),
-            )),
+            Some(Err(error)) => Err(Error::Transport(litellm_http::transport::Error::Network(
+                error.to_string(),
+            ))),
         }
     }
 
@@ -170,9 +162,7 @@ impl ResponsesWebSocketConnection {
         let mut socket = self.socket.lock().await;
         if let Some(socket) = socket.as_mut() {
             socket.close(None).await.map_err(|error| {
-                Error::Transport(litellm_llms::custom_httpx::transport::Error::Network(
-                    error.to_string(),
-                ))
+                Error::Transport(litellm_http::transport::Error::Network(error.to_string()))
             })?;
         }
         *socket = None;

litellm-rust/crates/core/tests/azure_document_intelligence_ocr.rs

@@ -1,10 +1,12 @@
 use litellm_host::event::{CallEvent, MachineEvent};
-use litellm_llms::base_llm::ocr::error::Error;
+use litellm_llms::base_llm::ocr::{error::Error, settings::OcrSettings};
 use rstest::rstest;
 use serde_json::{Value, json};
 
 use super::{
-    test_support::{MockResponse, mock_server, perform_ocr, perform_ocr_with, wire_request},
+    test_support::{
+        MockResponse, mock_server, ocr_client, perform_ocr, perform_ocr_with, wire_request,
+    },
     wire::{OcrWireRequest, decode_request},
 };
 use crate::ocr::route::LocalOcrHost;
@@ -200,6 +202,42 @@ async fn immediate_response_normalizes_pages_and_preserves_native() {
     );
 }
 
+#[tokio::test]
+async fn client_settings_choose_the_api_version_and_the_inch_to_pixel_dpi() {
+    let (base, seen, server) = mock_server(vec![MockResponse::json(json!({
+        "status":"succeeded",
+        "analyzeResult":{"pages":[{"pageNumber":1,"width":8.5,"height":11,"unit":"inch"}]}
+    }))])
+    .await;
+    let client = ocr_client().with_settings(OcrSettings {
+        document_intelligence_api_version: "2099-01-01".into(),
+        document_intelligence_dpi: 72,
+        ..OcrSettings::default()
+    });
+
+    let result = crate::ocr::client::perform(
+        &client,
+        wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({})),
+    )
+    .await
+    .unwrap();
+    server.await.unwrap();
+
+    let target = seen.lock().unwrap()[0]
+        .split_whitespace()
+        .nth(1)
+        .unwrap()
+        .to_string();
+    assert_eq!(
+        query_value(&format!("{base}{target}"), "api-version").as_deref(),
+        Some("2099-01-01")
+    );
+    assert_eq!(
+        serde_json::to_value(&result.pages[0].dimensions).unwrap(),
+        json!({"width":612,"height":792,"dpi":72})
+    );
+}
+
 #[tokio::test]
 async fn accepted_response_polls_to_success_with_only_credentials() {
     let operation = json!({"status":"succeeded","analyzeResult":{"pages":[]}});
@@ -425,13 +463,19 @@ async fn polling_deadline_bounds_retry_delay() {
         },
     ])
     .await;
-    let mut request = wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({}));
-    request.transport.poll_timeout = std::time::Duration::from_millis(100);
+    let request = wire_request("azure_ai/doc-intelligence/prebuilt-read", &base, json!({}));
+    let client = ocr_client().with_settings(OcrSettings {
+        poll_timeout: std::time::Duration::from_millis(100),
+        ..OcrSettings::default()
+    });
 
-    let error = tokio::time::timeout(std::time::Duration::from_secs(1), perform_ocr(request))
-        .await
-        .unwrap()
-        .unwrap_err();
+    let error = tokio::time::timeout(
+        std::time::Duration::from_secs(1),
+        crate::ocr::client::perform(&client, request),
+    )
+    .await
+    .unwrap()
+    .unwrap_err();
     server.await.unwrap();
     assert!(error.to_string().contains("timed out"));
 }

litellm-rust/crates/core/tests/ocr.rs

@@ -6,16 +6,15 @@ use litellm_host::{
     host::{Host, HostOp, HostResult},
     machine::{HostFailure, Machine, MachineStep},
 };
-use litellm_http::{HttpClientPool, HttpSettings, Resolution};
-use litellm_llms::{
-    base_llm::ocr::{
-        error::Error as OcrError,
-        transformation::{LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig},
-    },
-    custom_httpx::{
-        llm_http_handler::OcrClient,
-        media::{PublicDnsResolver, UrlPolicy},
-    },
+use litellm_http::{
+    HttpClientPool, HttpSettings, Resolution,
+    media::{PublicDnsResolver, UrlPolicy},
+};
+use litellm_llms::base_llm::ocr::{
+    error::Error as OcrError,
+    handler::OcrClient,
+    settings::OcrSettings,
+    transformation::{LiteLLMOcrResponse, OCR_RESPONSE_MAX_BYTES, OcrTransportConfig},
 };
 use rstest::rstest;
 use serde_json::{Value, json};
@@ -175,6 +174,43 @@ async fn facade_retains_native_response_when_requested() {
     );
 }
 
+#[rstest]
+#[case::plain_key(&[("MISTRAL_API_KEY", "plain")], "plain")]
+#[case::azure_key_wins(&[("MISTRAL_AZURE_API_KEY", "azure"), ("MISTRAL_API_KEY", "plain")], "azure")]
+#[case::empty_azure_key_falls_through(&[("MISTRAL_AZURE_API_KEY", ""), ("MISTRAL_API_KEY", "plain")], "plain")]
+#[tokio::test]
+async fn mistral_env_fallbacks_follow_python_through_the_injected_secret_source(
+    #[case] secrets: &'static [(&'static str, &'static str)],
+    #[case] expected_key: &str,
+) {
+    let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
+    let secret_base = base.clone();
+    let client = ocr_client().with_secrets(Arc::new(move |name: &str| match name {
+        "MISTRAL_AZURE_API_BASE" => Some(secret_base.clone()),
+        "MISTRAL_API_BASE" => Some("http://127.0.0.1:9/never-read".into()),
+        _ => secrets
+            .iter()
+            .find(|(key, _)| *key == name)
+            .map(|(_, value)| value.to_string()),
+    }));
+    let request = decode_request(OcrWireRequest {
+        model: "mistral/model".into(),
+        document: json!({"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}),
+        api_key: None,
+        api_base: None,
+        custom_llm_provider: None,
+        extra_headers: None,
+        optional_params: Default::default(),
+        input_sources: Default::default(),
+        timeout_seconds: Some(2.0),
+    })
+    .unwrap();
+
+    crate::ocr::client::perform(&client, request).await.unwrap();
+    server.await.unwrap();
+    assert!(seen.lock().unwrap()[0].contains(&format!("authorization: Bearer {expected_key}")));
+}
+
 #[tokio::test]
 async fn ocr_client_uses_the_injected_http_pool_configuration() {
     let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
@@ -187,6 +223,8 @@ async fn ocr_client_uses_the_injected_http_pool_configuration() {
         &Resolution::from(&settings).config,
         UrlPolicy::default(),
         VertexAuth::default(),
+        OcrSettings::default(),
+        Arc::new(litellm_core_utils::settings::ProcessEnvironment),
     )
     .unwrap();
     crate::ocr::client::perform(&client, wire_request("mistral/model", &base, json!({})))
@@ -624,7 +662,7 @@ async fn read_bounded_response(response: Vec<u8>, limit: usize) -> Result<bytes:
         .unwrap();
     let result = tokio::time::timeout(
         std::time::Duration::from_secs(2),
-        litellm_llms::custom_httpx::llm_http_handler::read_response_bytes(response, limit),
+        litellm_llms::base_llm::ocr::handler::read_response_bytes(response, limit),
     )
     .await;
     server.abort();
@@ -676,10 +714,7 @@ async fn oversized_error_retains_http_status_and_bounded_diagnostics_without_dra
         .await
         .unwrap_err();
     match error {
-        OcrError::Transport(litellm_llms::custom_httpx::transport::Error::Http {
-            status,
-            body,
-        }) => {
+        OcrError::Transport(litellm_http::transport::Error::Http { status, body }) => {
             assert_eq!(status, 429);
             assert_eq!(body, prefix);
         }

litellm-rust/crates/core/tests/ocr/support.rs

@@ -2,9 +2,10 @@ use std::sync::{Arc, Mutex};
 
 use futures_util::future::BoxFuture;
 use litellm_host::event::WireRequest;
-use litellm_llms::{
-    base_llm::ocr::{error::Error, transformation::LiteLLMOcrResponse},
-    custom_httpx::llm_http_handler::{CallHooks, OcrClient},
+use litellm_llms::base_llm::ocr::{
+    error::Error,
+    handler::{CallHooks, OcrClient},
+    transformation::LiteLLMOcrResponse,
 };
 use serde_json::{Value, json};
 use tokio::{

litellm-rust/crates/core/tests/vertex_ai_ocr.rs

@@ -1,8 +1,8 @@
 use litellm_auth::InputSource;
-use litellm_llms::base_llm::ocr::transformation::OcrResponseFormat;
+use litellm_llms::base_llm::ocr::{settings::OcrSettings, transformation::OcrResponseFormat};
 use serde_json::{Value, json};
 
-use super::test_support::{MockResponse, mock_server, perform_ocr, wire_request};
+use super::test_support::{MockResponse, mock_server, ocr_client, perform_ocr, wire_request};
 
 fn request_body(request: &str) -> Value {
     serde_json::from_str(request.split_once("\r\n\r\n").unwrap().1).unwrap()
@@ -48,6 +48,27 @@ async fn facade_executes_vertex_mistral_with_resolved_project_and_location() {
     );
 }
 
+#[tokio::test]
+async fn configured_project_and_location_apply_when_the_call_sets_neither() {
+    let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;
+    let client = ocr_client().with_settings(OcrSettings {
+        vertex_project: Some("configured-project".into()),
+        vertex_location: Some("europe-west4".into()),
+        ..OcrSettings::default()
+    });
+
+    crate::ocr::client::perform(
+        &client,
+        wire_request("vertex_ai/mistral-ocr-maas", &base, json!({})),
+    )
+    .await
+    .unwrap();
+    server.await.unwrap();
+    assert!(seen.lock().unwrap()[0].starts_with(
+        "POST /v1/projects/configured-project/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict "
+    ));
+}
+
 #[tokio::test]
 async fn supplied_authorization_is_forwarded_without_a_static_token() {
     let (base, seen, server) = mock_server(vec![MockResponse::json(json!({"pages":[]}))]).await;

litellm-rust/crates/http/Cargo.toml

@@ -5,12 +5,19 @@ edition.workspace = true
 license.workspace = true
 repository.workspace = true
 
+[features]
+test-support = []
+
 [dependencies]
 http.workspace = true
+litellm-core-utils.workspace = true
 hyper-util.workspace = true
 reqwest.workspace = true
 rustls.workspace = true
+serde_json.workspace = true
 thiserror.workspace = true
+tokio.workspace = true
+veil.workspace = true
 webpki-roots.workspace = true
 
 [dev-dependencies]

litellm-rust/crates/http/src/config.rs

@@ -6,6 +6,7 @@ use std::{
 
 use crate::{
     error::Error,
+    proxy::EnvironmentProxies,
     settings::{HttpSettings, SslVerify, TcpKeepalive},
     tls::{CipherSelection, KeyExchangeGroup, Tls12CipherSuite, Unsupported},
 };
@@ -26,7 +27,7 @@ pub struct HttpClientConfig {
     pub force_ipv4: bool,
     pub http2: bool,
     pub user_agent: Option<String>,
-    pub trust_proxy_env: bool,
+    pub proxies: EnvironmentProxies,
     pub connect_timeout: Duration,
     pub tcp_keepalive: Option<TcpKeepalive>,
     pub pool_idle_timeout: Duration,
@@ -72,7 +73,11 @@ impl From<&HttpSettings> for Resolution {
                 force_ipv4: settings.force_ipv4,
                 http2: settings.http2,
                 user_agent: settings.user_agent.clone(),
-                trust_proxy_env: settings.trust_proxy_env,
+                proxies: if settings.trust_proxy_env {
+                    settings.proxies.clone()
+                } else {
+                    EnvironmentProxies::default()
+                },
                 connect_timeout: settings.connect_timeout,
                 tcp_keepalive: settings.tcp_keepalive,
                 pool_idle_timeout: settings.pool_idle_timeout,
@@ -111,11 +116,11 @@ impl TryFrom<&HttpClientConfig> for reqwest::ClientBuilder {
             Some(agent) => with_protocol.user_agent(agent),
             None => with_protocol,
         };
-        Ok(if config.trust_proxy_env {
-            with_agent
-        } else {
-            with_agent.no_proxy()
-        })
+        Ok(config
+            .proxies
+            .reqwest_proxies()
+            .into_iter()
+            .fold(with_agent.no_proxy(), reqwest::ClientBuilder::proxy))
     }
 }
 
@@ -227,6 +232,25 @@ mod tests {
         );
     }
 
+    fn proxies() -> EnvironmentProxies {
+        EnvironmentProxies::from_environment(&|name: &str| {
+            (name == "HTTPS_PROXY").then(|| "http://proxy.corp:3128".to_string())
+        })
+    }
+
+    #[test]
+    fn proxies_are_dropped_when_the_transport_does_not_trust_the_environment() {
+        let settings = HttpSettings {
+            trust_proxy_env: false,
+            proxies: proxies(),
+            ..HttpSettings::default()
+        };
+        assert_eq!(
+            Resolution::from(&settings).config.proxies,
+            EnvironmentProxies::default()
+        );
+    }
+
     #[test]
     fn connection_settings_carry_over_unchanged() {
         let keepalive = TcpKeepalive {
@@ -240,6 +264,7 @@ mod tests {
             http2: true,
             user_agent: Some("litellm/1.0".into()),
             trust_proxy_env: true,
+            proxies: proxies(),
             connect_timeout: Duration::from_secs(7),
             tcp_keepalive: Some(keepalive),
             pool_idle_timeout: Duration::from_secs(45),
@@ -256,7 +281,7 @@ mod tests {
                 force_ipv4: true,
                 http2: true,
                 user_agent: Some("litellm/1.0".into()),
-                trust_proxy_env: true,
+                proxies: proxies(),
                 connect_timeout: Duration::from_secs(7),
                 tcp_keepalive: Some(keepalive),
                 pool_idle_timeout: Duration::from_secs(45),

litellm-rust/crates/http/src/lib.rs

@@ -1,9 +1,12 @@
 mod config;
 mod error;
+pub mod media;
 mod pool;
 mod proxy;
+pub mod request;
 mod settings;
 mod tls;
+pub mod transport;
 
 pub use config::{HttpClientConfig, Resolution, Verify};
 pub use error::Error;

litellm-rust/crates/http/src/media.rs

@@ -7,12 +7,13 @@ use std::{
     time::Duration,
 };
 
-use litellm_http::{ClientVariant, EnvironmentProxies, HttpClientConfig, HttpClientPool};
 use reqwest::{
     Url,
     dns::{Addrs, Name, Resolve, Resolving},
 };
 
+use crate::{ClientVariant, HttpClientConfig, HttpClientPool};
+
 #[derive(Debug, thiserror::Error)]
 pub enum Error {
     #[error("media URL rejected by network policy")]
@@ -32,7 +33,7 @@ pub enum Error {
     #[error("media download timed out")]
     Timeout,
     #[error("{0}")]
-    Transport(#[from] crate::custom_httpx::transport::Error),
+    Transport(#[from] crate::transport::Error),
 }
 
 #[derive(Clone, Debug, PartialEq, Eq)]
@@ -101,13 +102,8 @@ impl MediaFetcher {
         pool: &HttpClientPool,
         config: &HttpClientConfig,
         url_policy: UrlPolicy,
-    ) -> Result<Self, litellm_http::Error> {
-        let uses_proxy: ProxyMatch = if config.trust_proxy_env {
-            let proxies = EnvironmentProxies::from_environment();
-            Arc::new(move |url| proxies.apply_to(url))
-        } else {
-            Arc::new(|_| false)
-        };
+    ) -> Result<Self, crate::Error> {
+        let uses_proxy: ProxyMatch = Arc::new(config.proxies.matcher());
         Self::with_resolution(
             pool,
             config,
@@ -123,7 +119,7 @@ impl MediaFetcher {
         url_policy: UrlPolicy,
         address_resolver: Arc<dyn AddressResolver>,
         uses_proxy: ProxyMatch,
-    ) -> Result<Self, litellm_http::Error> {
+    ) -> Result<Self, crate::Error> {
         Ok(Self {
             pinned: pool.client(config, ClientVariant::Media)?,
             unpinned: pool.client(config, ClientVariant::UnpinnedMedia)?,
@@ -168,7 +164,7 @@ impl MediaFetcher {
                 .get(url.clone())
                 .send()
                 .await
-                .map_err(crate::custom_httpx::transport::Error::from)?;
+                .map_err(crate::transport::Error::from)?;
             if response.status().is_redirection() {
                 if redirects_followed == policy.max_redirects {
                     return Err(Error::TooManyRedirects);
@@ -199,7 +195,7 @@ impl MediaFetcher {
             while let Some(chunk) = response
                 .chunk()
                 .await
-                .map_err(crate::custom_httpx::transport::Error::from)?
+                .map_err(crate::transport::Error::from)?
             {
                 enforce_download_size(bytes.len() as u64 + chunk.len() as u64, policy.max_bytes)?;
                 bytes.extend_from_slice(&chunk);
@@ -249,7 +245,7 @@ impl MediaFetcher {
             .address_resolver
             .resolve(host, port)
             .await
-            .map_err(|error| crate::custom_httpx::transport::Error::Network(error.to_string()))?;
+            .map_err(|error| crate::transport::Error::Network(error.to_string()))?;
         validate_addresses(&addresses)
     }
 }
@@ -350,13 +346,13 @@ impl Resolve for PublicDnsResolver {
 mod tests {
     use std::collections::HashSet;
 
-    use litellm_http::{HttpSettings, Resolution};
     use tokio::{
         io::{AsyncReadExt, AsyncWriteExt},
         net::TcpListener,
     };
 
     use super::*;
+    use crate::{HttpSettings, Resolution};
 
     async fn serve(response: &'static [u8]) -> (Url, tokio::task::JoinHandle<()>) {
         let listener = TcpListener::bind("127.0.0.1:0")
@@ -443,10 +439,7 @@ mod tests {
         url_policy: UrlPolicy,
         uses_proxy: bool,
     ) -> MediaFetcher {
-        let direct = HttpClientConfig {
-            trust_proxy_env: false,
-            ..Resolution::from(&HttpSettings::default()).config
-        };
+        let direct = Resolution::from(&HttpSettings::default()).config;
         MediaFetcher::with_resolution(
             &HttpClientPool::new(Arc::new(LoopbackDnsResolver(pinned_address))),
             &direct,

litellm-rust/crates/http/src/pool.rs

@@ -6,7 +6,7 @@ use std::{
 
 use reqwest::dns::Resolve;
 
-use crate::{config::HttpClientConfig, error::Error};
+use crate::{config::HttpClientConfig, error::Error, proxy::EnvironmentProxies};
 
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
 pub enum ClientVariant {
@@ -52,7 +52,7 @@ impl HttpClientPool {
         let effective = match variant {
             ClientVariant::Media => HttpClientConfig {
                 client_certificate: None,
-                trust_proxy_env: false,
+                proxies: EnvironmentProxies::default(),
                 ..config.clone()
             },
             ClientVariant::UnpinnedMedia => HttpClientConfig {
@@ -138,6 +138,13 @@ mod tests {
         }
     }
 
+    fn proxied_through(proxy: &str) -> EnvironmentProxies {
+        let proxy = proxy.to_owned();
+        EnvironmentProxies::from_environment(&move |name: &str| {
+            (name == "HTTP_PROXY").then(|| proxy.clone())
+        })
+    }
+
     async fn serve(
         status_line: &'static str,
     ) -> (SocketAddr, Arc<AtomicUsize>, Arc<Mutex<Vec<String>>>) {
@@ -202,6 +209,50 @@ mod tests {
         assert_eq!(connections.load(Ordering::SeqCst), 3);
     }
 
+    #[tokio::test]
+    async fn provider_clients_route_through_the_resolved_proxy_not_the_process_environment() {
+        let (proxy, connections, requests) = serve("HTTP/1.1 204 No Content").await;
+        let config = HttpClientConfig {
+            proxies: proxied_through(&format!("http://user:secret@{proxy}")),
+            ..config("a")
+        };
+        let response = get(
+            &pool(),
+            &config,
+            ClientVariant::Provider,
+            "http://upstream.invalid/v1/ocr",
+        )
+        .await;
+        assert_eq!(response.status(), 204);
+        assert_eq!(connections.load(Ordering::SeqCst), 1);
+        let request = requests.lock().unwrap().concat();
+        assert!(request.starts_with("GET http://upstream.invalid/v1/ocr HTTP/1.1"));
+        assert!(request.contains("proxy-authorization: Basic dXNlcjpzZWNyZXQ="));
+    }
+
+    #[tokio::test]
+    async fn no_proxy_hosts_bypass_the_resolved_proxy() {
+        let (upstream, _, _) = serve("HTTP/1.1 204 No Content").await;
+        let (proxy, proxy_connections, _) = serve("HTTP/1.1 502 Bad Gateway").await;
+        let config = HttpClientConfig {
+            proxies: EnvironmentProxies::from_environment(&move |name: &str| match name {
+                "HTTP_PROXY" => Some(format!("http://{proxy}")),
+                "NO_PROXY" => Some("127.0.0.1".into()),
+                _ => None,
+            }),
+            ..config("a")
+        };
+        let response = get(
+            &pool(),
+            &config,
+            ClientVariant::Provider,
+            &format!("http://{upstream}/v1/ocr"),
+        )
+        .await;
+        assert_eq!(response.status(), 204);
+        assert_eq!(proxy_connections.load(Ordering::SeqCst), 0);
+    }
+
     #[tokio::test]
     async fn expired_clients_are_rebuilt() {
         let (address, connections, _) = serve("HTTP/1.1 204 No Content").await;
@@ -220,9 +271,12 @@ mod tests {
         let (address, connections, _) = serve("HTTP/1.1 204 No Content").await;
         let pool = HttpClientPool::new(Arc::new(FixedResolver(address)));
         let url = format!("http://media.invalid:{}/doc", address.port());
-        for trust_proxy_env in [true, false] {
+        for proxies in [
+            proxied_through("http://proxy.invalid:3128"),
+            EnvironmentProxies::default(),
+        ] {
             let config = HttpClientConfig {
-                trust_proxy_env,
+                proxies,
                 ..config("a")
             };
             get(&pool, &config, ClientVariant::Media, &url).await;

litellm-rust/crates/http/src/proxy.rs

@@ -1,15 +1,164 @@
 use hyper_util::client::proxy::matcher::Matcher;
+use litellm_core_utils::settings::Lookup;
+use veil::Redact;
 
-pub struct EnvironmentProxies(Matcher);
+#[derive(Clone, Redact, Default, PartialEq, Eq, Hash)]
+pub struct EnvironmentProxies {
+    #[redact]
+    all: String,
+    #[redact]
+    http: String,
+    #[redact]
+    https: String,
+    no: String,
+}
 
 impl EnvironmentProxies {
-    pub fn from_environment() -> Self {
-        Self(Matcher::from_system())
+    pub fn from_environment(env: &impl Lookup) -> Self {
+        Self::resolve(env, cfg!(windows))
     }
 
-    pub fn apply_to(&self, url: &reqwest::Url) -> bool {
-        url.as_str()
-            .parse::<http::Uri>()
-            .is_ok_and(|uri| self.0.intercept(&uri).is_some())
+    fn resolve(env: &impl Lookup, names_ignore_case: bool) -> Self {
+        let lowercase_first = |upper: Option<&str>, lower: &str| {
+            env.get(lower)
+                .or_else(|| upper.and_then(|name| env.truthy(name)))
+                .unwrap_or_default()
+        };
+        let is_cgi = env.get("REQUEST_METHOD").is_some();
+        Self {
+            all: lowercase_first(Some("ALL_PROXY"), "all_proxy"),
+            http: if is_cgi && names_ignore_case {
+                String::new()
+            } else {
+                lowercase_first((!is_cgi).then_some("HTTP_PROXY"), "http_proxy")
+            },
+            https: lowercase_first(Some("HTTPS_PROXY"), "https_proxy"),
+            no: lowercase_first(Some("NO_PROXY"), "no_proxy"),
+        }
+    }
+
+    pub(crate) fn matcher(&self) -> impl Fn(&reqwest::Url) -> bool + Send + Sync + use<> {
+        let matcher = Matcher::builder()
+            .all(self.all.clone())
+            .http(self.http.clone())
+            .https(self.https.clone())
+            .no(self.no.clone())
+            .build();
+        move |url| {
+            url.as_str()
+                .parse::<http::Uri>()
+                .is_ok_and(|uri| matcher.intercept(&uri).is_some())
+        }
+    }
+
+    pub(crate) fn reqwest_proxies(&self) -> Vec<reqwest::Proxy> {
+        let no_proxy = reqwest::NoProxy::from_string(&self.no);
+        [
+            reqwest::Proxy::http(self.http.as_str()),
+            reqwest::Proxy::https(self.https.as_str()),
+            reqwest::Proxy::all(self.all.as_str()),
+        ]
+        .into_iter()
+        .filter_map(Result::ok)
+        .map(|proxy| proxy.no_proxy(no_proxy.clone()))
+        .collect()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
+        move |name| {
+            values
+                .iter()
+                .find(|(key, _)| *key == name)
+                .map(|(_, value)| value.to_string())
+        }
+    }
+
+    fn url(value: &str) -> reqwest::Url {
+        reqwest::Url::parse(value).unwrap()
+    }
+
+    #[rstest]
+    #[case::http_only(&[("HTTP_PROXY", "http://proxy:3128")], "http://api.test/", true)]
+    #[case::http_proxy_skips_https(&[("HTTP_PROXY", "http://proxy:3128")], "https://api.test/", false)]
+    #[case::all_covers_https(&[("ALL_PROXY", "http://proxy:3128")], "https://api.test/", true)]
+    #[case::lowercase(&[("https_proxy", "http://proxy:3128")], "https://api.test/", true)]
+    #[case::no_proxy_bypass(&[("HTTPS_PROXY", "http://proxy:3128"), ("NO_PROXY", "api.test")], "https://api.test/", false)]
+    fn proxies_follow_the_injected_environment(
+        #[case] env: &'static [(&'static str, &'static str)],
+        #[case] target: &str,
+        #[case] expected: bool,
+    ) {
+        let proxies = EnvironmentProxies::from_environment(&env_of(env));
+        assert_eq!(proxies.matcher()(&url(target)), expected);
+    }
+
+    #[rstest]
+    #[case::lowercase_wins(&[("HTTPS_PROXY", "http://upper:3128"), ("https_proxy", "http://lower:3128")], &[("https_proxy", "http://lower:3128")])]
+    #[case::empty_uppercase_falls_through_to_lowercase(&[("HTTPS_PROXY", ""), ("https_proxy", "http://lower:3128")], &[("https_proxy", "http://lower:3128")])]
+    #[case::empty_uppercase_alone_is_unset(&[("HTTPS_PROXY", "")], &[])]
+    #[case::empty_lowercase_clears_the_uppercase_value(&[("HTTPS_PROXY", "http://upper:3128"), ("https_proxy", "")], &[])]
+    #[case::lowercase_no_proxy_wins(&[("NO_PROXY", "upper.test"), ("no_proxy", "lower.test")], &[("no_proxy", "lower.test")])]
+    #[case::cgi_forgets_the_client_settable_http_proxy(&[("REQUEST_METHOD", "GET"), ("HTTP_PROXY", "http://attacker:3128")], &[])]
+    #[case::cgi_keeps_lowercase_http_proxy(&[("REQUEST_METHOD", "GET"), ("HTTP_PROXY", "http://attacker:3128"), ("http_proxy", "http://lower:3128")], &[("http_proxy", "http://lower:3128")])]
+    #[case::cgi_keeps_every_other_variable(&[("REQUEST_METHOD", "GET"), ("HTTPS_PROXY", "http://proxy:3128"), ("ALL_PROXY", "http://all:3128"), ("NO_PROXY", "internal.test")], &[("HTTPS_PROXY", "http://proxy:3128"), ("ALL_PROXY", "http://all:3128"), ("NO_PROXY", "internal.test")])]
+    fn variables_resolve_like_urllib_getproxies_environment(
+        #[case] env: &'static [(&'static str, &'static str)],
+        #[case] equivalent: &'static [(&'static str, &'static str)],
+    ) {
+        assert_eq!(
+            EnvironmentProxies::from_environment(&env_of(env)),
+            EnvironmentProxies::from_environment(&env_of(equivalent))
+        );
+    }
+
+    #[test]
+    fn cgi_drops_http_proxy_entirely_where_variable_names_ignore_case() {
+        let windows_env = |name: &str| match name.to_ascii_uppercase().as_str() {
+            "REQUEST_METHOD" => Some("GET".to_string()),
+            "HTTP_PROXY" => Some("http://attacker:3128".to_string()),
+            "HTTPS_PROXY" => Some("http://proxy:3128".to_string()),
+            _ => None,
+        };
+        let proxies = EnvironmentProxies::resolve(&windows_env, true);
+        assert!(!proxies.matcher()(&url("http://api.test/")));
+        assert!(proxies.matcher()(&url("https://api.test/")));
+        assert!(EnvironmentProxies::resolve(&windows_env, false).matcher()(
+            &url("http://api.test/")
+        ));
+    }
+
+    #[test]
+    fn a_cgi_request_still_proxies_https_through_the_configured_proxy() {
+        let proxies = EnvironmentProxies::from_environment(&env_of(&[
+            ("REQUEST_METHOD", "GET"),
+            ("HTTPS_PROXY", "http://proxy:3128"),
+        ]));
+        assert!(proxies.matcher()(&url("https://api.test/")));
+    }
+
+    #[test]
+    fn debug_output_hides_proxy_credentials_but_shows_which_variables_are_set() {
+        let proxies = EnvironmentProxies::from_environment(&env_of(&[
+            ("HTTPS_PROXY", "http://operator:hunter2@proxy.corp:3128"),
+            ("NO_PROXY", "internal.test"),
+        ]));
+        let debug = format!("{proxies:?}");
+        assert!(!debug.contains("hunter2") && !debug.contains("operator"));
+        assert!(debug.contains("internal.test"));
+        assert_ne!(debug, format!("{:?}", EnvironmentProxies::default()));
+    }
+
+    #[test]
+    fn an_empty_environment_proxies_nothing() {
+        let proxies = EnvironmentProxies::from_environment(&env_of(&[]));
+        assert_eq!(proxies, EnvironmentProxies::default());
+        assert!(proxies.reqwest_proxies().is_empty());
     }
 }

litellm-rust/crates/http/src/request.rs

@@ -13,20 +13,12 @@ use serde_json::{Map, Value};
 /// before truncation, so provider bodies are bounded and data-minimized.
 const UPSTREAM_ERROR_BODY_MAX_CHARS: usize = 256;
 
-#[allow(
-    dead_code,
-    reason = "used by the OCR architecture in the next stacked PR"
-)]
 pub enum HeaderPolicy<'a> {
     All,
     Only(&'a [&'a str]),
     Except(&'a [&'a str]),
 }
 
-#[allow(
-    dead_code,
-    reason = "used by the OCR architecture in the next stacked PR"
-)]
 pub fn with_headers(
     builder: reqwest::RequestBuilder,
     headers: &[(String, String)],
@@ -107,18 +99,6 @@ pub fn has_bearer_auth(headers: &[(String, String)]) -> bool {
     })
 }
 
-#[allow(
-    dead_code,
-    reason = "used by the OCR architecture in the next stacked PR"
-)]
-pub fn deserialize_optional_param<'de, D, T>(deserializer: D) -> Result<Option<Option<T>>, D::Error>
-where
-    D: serde::Deserializer<'de>,
-    T: serde::Deserialize<'de>,
-{
-    <Option<T> as serde::Deserialize>::deserialize(deserializer).map(Some)
-}
-
 #[cfg(test)]
 mod tests {
     use serde_json::json;

litellm-rust/crates/http/src/settings.rs

@@ -3,6 +3,10 @@ use std::{
     time::Duration,
 };
 
+use litellm_core_utils::settings::{Layer, Lookup, merge};
+
+use crate::proxy::EnvironmentProxies;
+
 #[derive(Clone, Debug, PartialEq, Eq, Hash)]
 pub enum SslVerify {
     Enabled,
@@ -42,41 +46,41 @@ pub struct HttpSettingsLayer {
     pub user_agent: Option<String>,
     pub tcp_keepalive: Option<TcpKeepalive>,
     pub pool_idle_timeout: Option<Duration>,
+    pub proxies: Option<EnvironmentProxies>,
 }
 
 impl HttpSettingsLayer {
-    pub fn from_environment(env: &(dyn Fn(&str) -> Option<String> + Sync)) -> Self {
-        let enabled = |name: &str| {
-            env(name)
-                .is_some_and(|value| value.trim().eq_ignore_ascii_case("true"))
-                .then_some(true)
-        };
-        let number = |name: &str| env(name).and_then(|value| value.trim().parse::<u32>().ok());
+    pub fn from_environment(env: &impl Lookup) -> Self {
         let seconds = |name: &str, default: u32| {
-            Duration::from_secs(u64::from(number(name).unwrap_or(default)))
+            Duration::from_secs(u64::from(env.parsed::<u32>(name).unwrap_or(default)))
         };
         Self {
-            ssl_verify: env("SSL_VERIFY").map(|value| SslVerify::parse(&value)),
-            ssl_cert_file: env("SSL_CERT_FILE").map(PathBuf::from),
-            ssl_certificate: env("SSL_CERTIFICATE").map(PathBuf::from),
-            ssl_security_level: env("SSL_SECURITY_LEVEL"),
-            ssl_ecdh_curve: env("SSL_ECDH_CURVE"),
+            ssl_verify: env.get("SSL_VERIFY").map(|value| SslVerify::parse(&value)),
+            ssl_cert_file: env.truthy("SSL_CERT_FILE").map(PathBuf::from),
+            ssl_certificate: env.get("SSL_CERTIFICATE").map(PathBuf::from),
+            ssl_security_level: env.get("SSL_SECURITY_LEVEL"),
+            ssl_ecdh_curve: env.get("SSL_ECDH_CURVE"),
             force_ipv4: None,
-            http2: enabled("LITELLM_HTTP2"),
-            aiohttp_trust_env: enabled("AIOHTTP_TRUST_ENV"),
-            disable_aiohttp_trust_env: enabled("DISABLE_AIOHTTP_TRUST_ENV"),
-            disable_aiohttp_transport: enabled("DISABLE_AIOHTTP_TRANSPORT"),
-            user_agent: env("LITELLM_USER_AGENT"),
-            tcp_keepalive: enabled("AIOHTTP_SO_KEEPALIVE").map(|_| TcpKeepalive {
+            http2: env.enabled("LITELLM_HTTP2"),
+            aiohttp_trust_env: env.enabled("AIOHTTP_TRUST_ENV"),
+            disable_aiohttp_trust_env: env.enabled("DISABLE_AIOHTTP_TRUST_ENV"),
+            disable_aiohttp_transport: env.enabled("DISABLE_AIOHTTP_TRANSPORT"),
+            user_agent: env.get("LITELLM_USER_AGENT"),
+            tcp_keepalive: env.enabled("AIOHTTP_SO_KEEPALIVE").map(|_| TcpKeepalive {
                 idle: seconds("AIOHTTP_TCP_KEEPIDLE", 60),
                 interval: seconds("AIOHTTP_TCP_KEEPINTVL", 30),
-                retries: number("AIOHTTP_TCP_KEEPCNT").unwrap_or(5),
+                retries: env.parsed("AIOHTTP_TCP_KEEPCNT").unwrap_or(5),
             }),
-            pool_idle_timeout: number("AIOHTTP_KEEPALIVE_TIMEOUT")
+            pool_idle_timeout: env
+                .parsed::<u32>("AIOHTTP_KEEPALIVE_TIMEOUT")
                 .map(|timeout| Duration::from_secs(u64::from(timeout))),
+            proxies: Some(EnvironmentProxies::from_environment(env))
+                .filter(|proxies| *proxies != EnvironmentProxies::default()),
         }
     }
+}
 
+impl Layer for HttpSettingsLayer {
     fn or(self, lower: Self) -> Self {
         Self {
             ssl_verify: self.ssl_verify.or(lower.ssl_verify),
@@ -96,6 +100,7 @@ impl HttpSettingsLayer {
             user_agent: self.user_agent.or(lower.user_agent),
             tcp_keepalive: self.tcp_keepalive.or(lower.tcp_keepalive),
             pool_idle_timeout: self.pool_idle_timeout.or(lower.pool_idle_timeout),
+            proxies: self.proxies.or(lower.proxies),
         }
     }
 }
@@ -111,6 +116,7 @@ pub struct HttpSettings {
     pub http2: bool,
     pub user_agent: Option<String>,
     pub trust_proxy_env: bool,
+    pub proxies: EnvironmentProxies,
     pub connect_timeout: Duration,
     pub tcp_keepalive: Option<TcpKeepalive>,
     pub pool_idle_timeout: Duration,
@@ -128,6 +134,7 @@ impl Default for HttpSettings {
             http2: false,
             user_agent: None,
             trust_proxy_env: true,
+            proxies: EnvironmentProxies::default(),
             connect_timeout: Duration::from_secs(10),
             tcp_keepalive: None,
             pool_idle_timeout: Duration::from_secs(120),
@@ -139,10 +146,7 @@ impl HttpSettings {
     pub fn from_layers(
         highest_precedence_first: impl IntoIterator<Item = HttpSettingsLayer>,
     ) -> Self {
-        let merged = highest_precedence_first
-            .into_iter()
-            .reduce(HttpSettingsLayer::or)
-            .unwrap_or_default();
+        let merged = merge(highest_precedence_first);
         let defaults = Self::default();
         let http2 = merged.http2.unwrap_or(defaults.http2);
         Self {
@@ -164,6 +168,7 @@ impl HttpSettings {
             pool_idle_timeout: merged
                 .pool_idle_timeout
                 .unwrap_or(defaults.pool_idle_timeout),
+            proxies: merged.proxies.unwrap_or_default(),
             ..defaults
         }
     }
@@ -190,9 +195,7 @@ mod tests {
         None
     }
 
-    fn env_of(
-        values: &'static [(&'static str, &'static str)],
-    ) -> impl Fn(&str) -> Option<String> + Sync {
+    fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
         move |name| {
             values
                 .iter()

litellm-rust/crates/http/src/transport.rs

@@ -46,11 +46,8 @@ mod tests {
             .send()
             .await
             .expect_err("invalid port");
-        let error = crate::custom_httpx::transport::Error::from_reqwest_before_dispatch(error);
-        assert!(matches!(
-            error,
-            crate::custom_httpx::transport::Error::Connect(_)
-        ));
+        let error = crate::transport::Error::from_reqwest_before_dispatch(error);
+        assert!(matches!(error, crate::transport::Error::Connect(_)));
         assert!(!error.to_string().contains("secret"));
         assert!(!error.to_string().contains("private"));
     }
@@ -76,7 +73,7 @@ mod tests {
             .await
             .expect_err("nothing listens on the port");
         let root_cause = root_cause(&error).expect("reqwest reports a cause");
-        let message = crate::custom_httpx::transport::Error::from(error).to_string();
+        let message = crate::transport::Error::from(error).to_string();
         assert!(message.contains(&root_cause), "{message}");
         assert!(!message.contains("secret"));
     }
@@ -105,8 +102,8 @@ mod tests {
         let error = response.expect_err("server does not respond");
         assert!(error.is_timeout());
         assert!(matches!(
-            crate::custom_httpx::transport::Error::from_reqwest_before_dispatch(error),
-            crate::custom_httpx::transport::Error::Network(_)
+            crate::transport::Error::from_reqwest_before_dispatch(error),
+            crate::transport::Error::Network(_)
         ));
     }
 }

litellm-rust/crates/llms/AGENTS.md

@@ -1,4 +1,4 @@
-litellm-llms mirrors `litellm/llms/`: base config traits, provider transformations, and the `custom_httpx` handlers. See `../core/AGENTS.md` for how the crates layer.
+litellm-llms mirrors `litellm/llms/`: base config traits, provider transformations, and the OCR request handler in `base_llm/ocr/handler.rs`. Transport code (clients, media fetching, header helpers, transport errors) lives in `litellm-http`. See `../core/AGENTS.md` for how the crates layer.
 
 ## Python/Rust transformation pairs
 

litellm-rust/crates/llms/Cargo.toml

@@ -6,7 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [features]
-test-support = []
+test-support = ["litellm-http/test-support"]
 
 [dependencies]
 litellm-types.workspace = true

litellm-rust/crates/llms/src/azure_ai/ocr/cohere_parse_transformation.rs

@@ -5,6 +5,7 @@ use crate::{
     base_llm::ocr::{
         document::{inline_remote_document, validate_inline_document},
         error::Error,
+        handler::OcrClient,
         transformation::{
             BaseOcrConfig, LiteLLMOcrResponse, OcrDocument, OcrRequestContext, OcrResponseFormat,
             PreparedOcrRequest,
@@ -13,7 +14,6 @@ use crate::{
     cohere::ocr::transformation::{
         CohereOptions, CohereParseConfig, CohereRequest, validate_document,
     },
-    custom_httpx::llm_http_handler::OcrClient,
 };
 
 #[derive(Default)]
@@ -53,7 +53,7 @@ impl BaseOcrConfig for AzureAICohereParseConfig {
     ) -> Result<String, Error> {
         let base = super::transformation::AzureAiOcrConfig::resolve_api_base(
             request.connection.api_base.as_deref(),
-            &crate::base_llm::ocr::transformation::credential_env,
+            &|name: &str| request.connection.secret(name),
         )?;
         self.get_complete_url(&base)
     }
@@ -108,7 +108,7 @@ impl BaseOcrConfig for AzureAICohereParseConfig {
     }
 
     fn validate_request_body(&self, body: &Value) -> Result<(), Error> {
-        let document = crate::custom_httpx::llm_http_handler::body_document(body)?;
+        let document = crate::base_llm::ocr::handler::body_document(body)?;
         validate_document(&document)?;
         validate_inline_document(&document)
     }

litellm-rust/crates/llms/src/azure_ai/ocr/common_utils.rs

@@ -3,7 +3,21 @@ use std::sync::OnceLock;
 use litellm_auth::{InputSource, Sourced};
 use litellm_auth_azure::{AzureAuthInputs, AzureAuthService};
 
-use crate::base_llm::ocr::{error::Error, transformation::OcrConnection};
+use crate::base_llm::ocr::{
+    error::Error,
+    transformation::{OcrConnection, PreparedOcrRequest},
+};
+
+pub(crate) fn azure_auth_inputs(request: &PreparedOcrRequest) -> Result<AzureAuthInputs, Error> {
+    Ok(AzureAuthInputs {
+        azure_ad_token_provider: request.azure_ad_token_provider.clone(),
+        ..AzureAuthInputs::from_sourced_optional_params(
+            &request.optional_params,
+            &request.input_sources,
+        )?
+    }
+    .or_configured_token_refresh(request.connection.settings.enable_azure_ad_token_refresh))
+}
 
 pub(super) async fn resolve_entra(
     config: &AzureAuthInputs,

litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs

@@ -14,24 +14,20 @@ use serde_json::{Map, Value};
 use serde_with::serde_as;
 use tokio::time::Instant;
 
-use crate::{
-    base_llm::ocr::{
-        document::InlineDocument,
-        error::Error,
-        transformation::{
-            BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES,
-            OCR_POLL_RETRY_SECS, OcrConnection, OcrCredentialInputs, OcrDocument, OcrPage,
-            OcrPageDimensions, OcrResponseContext, OcrResponseFormat, OcrUsageInfo,
-            PreparedOcrRequest, ResolvedOcrCredentials, credential_env,
-            decode_and_normalize_response, decode_response,
-        },
+use crate::base_llm::ocr::{
+    document::InlineDocument,
+    error::Error,
+    handler::{CallHooks, OcrClient, read_json_response},
+    settings::OcrSettings,
+    transformation::{
+        BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES,
+        OCR_POLL_RETRY_SECS, OcrConnection, OcrCredentialInputs, OcrDocument, OcrPage,
+        OcrPageDimensions, OcrResponseContext, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
+        ResolvedOcrCredentials, decode_and_normalize_response, decode_response,
     },
-    custom_httpx::llm_http_handler::{CallHooks, OcrClient, read_json_response},
 };
 
-const AZURE_DI_API_VERSION: &str = "2024-11-30";
 const AZURE_DI_SUBSCRIPTION_HEADER: &str = "Ocp-Apim-Subscription-Key";
-const AZURE_DI_DEFAULT_DPI: i64 = 96;
 const AZURE_DI_DEFAULT_WIDTH: f64 = 8.5;
 const AZURE_DI_DEFAULT_HEIGHT: f64 = 11.0;
 
@@ -178,15 +174,11 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
         request: &PreparedOcrRequest,
         _client: &OcrClient,
     ) -> Result<Self::Environment, Error> {
-        let config = AzureAuthInputs {
-            azure_ad_token_provider: request.azure_ad_token_provider.clone(),
-            ..AzureAuthInputs::from_sourced_optional_params(
-                &request.optional_params,
-                &request.input_sources,
-            )?
-        };
-        self.resolve_headers(&request.connection, &config, &credential_env)
-            .await
+        let config = crate::azure_ai::ocr::common_utils::azure_auth_inputs(request)?;
+        self.resolve_headers(&request.connection, &config, &|name: &str| {
+            request.connection.secret(name)
+        })
+        .await
     }
 
     fn get_complete_url(
@@ -196,9 +188,17 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
         _environment: &Self::Environment,
     ) -> Result<String, Error> {
         let endpoint = nonblank(request.connection.api_base.clone())
-            .or_else(|| nonblank(credential_env(AZURE_DI_ENDPOINT_ENV)))
+            .or_else(|| nonblank(request.connection.secret(AZURE_DI_ENDPOINT_ENV)))
             .ok_or_else(|| Error::Auth(litellm_auth::Error::ProviderAuthentication("Missing Azure Document Intelligence API Base - Set AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT or pass api_base".into())))?;
-        self.build_ocr_url(&endpoint, &request.model, optional_params)
+        self.build_ocr_url(
+            &endpoint,
+            &request.model,
+            optional_params,
+            &request
+                .connection
+                .settings
+                .document_intelligence_api_version,
+        )
     }
 
     fn transform_ocr_request(
@@ -217,12 +217,13 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
         raw_response: &[u8],
         request_format: OcrResponseFormat,
     ) -> Result<LiteLLMOcrResponse, Error> {
-        decode_and_normalize_response(
-            model,
-            raw_response,
-            request_format,
-            transform_completed_response,
-        )
+        decode_and_normalize_response(model, raw_response, request_format, |model, response| {
+            transform_completed_response(
+                model,
+                response,
+                OcrSettings::default().document_intelligence_dpi,
+            )
+        })
     }
 
     async fn async_transform_ocr_response(
@@ -243,7 +244,11 @@ impl BaseOcrConfig for AzureDocumentIntelligenceOcrConfig {
         .await?;
         Ok(LiteLLMOcrResponse {
             provider_native_response: decoded.native,
-            ..transform_completed_response(model, decoded.data)?
+            ..transform_completed_response(
+                model,
+                decoded.data,
+                context.connection.settings.document_intelligence_dpi,
+            )?
         })
     }
 }
@@ -356,6 +361,7 @@ fn build_request(document: OcrDocument) -> Result<DocumentIntelligenceRequest, E
 fn transform_completed_response(
     model: &str,
     response: AzureDocumentIntelligenceOperation,
+    dpi: i64,
 ) -> Result<LiteLLMOcrResponse, Error> {
     if response.status != Some(OperationStatus::Succeeded) {
         return Err(Error::OperationStatus(
@@ -369,7 +375,7 @@ fn transform_completed_response(
     let pages = result
         .pages
         .into_iter()
-        .map(transform_azure_page)
+        .map(|page| transform_azure_page(page, dpi))
         .collect::<Result<Vec<_>, _>>()?;
     let pages_processed = i64::try_from(pages.len()).map_err(|_| Error::NumericRange("pages"))?;
     Ok(LiteLLMOcrResponse {
@@ -384,7 +390,7 @@ fn transform_completed_response(
     })
 }
 
-fn transform_azure_page(page: AzureDocumentIntelligencePage) -> Result<OcrPage, Error> {
+fn transform_azure_page(page: AzureDocumentIntelligencePage, dpi: i64) -> Result<OcrPage, Error> {
     let index = page
         .page_number
         .unwrap_or(1)
@@ -394,6 +400,7 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage) -> Result<OcrPage,
         page.width.unwrap_or(AZURE_DI_DEFAULT_WIDTH),
         page.height.unwrap_or(AZURE_DI_DEFAULT_HEIGHT),
         page.unit.as_deref().unwrap_or("inch"),
+        dpi,
     )?;
     let markdown = page
         .lines
@@ -409,16 +416,17 @@ fn transform_azure_page(page: AzureDocumentIntelligencePage) -> Result<OcrPage,
     })
 }
 
-fn convert_dimensions(width: f64, height: f64, unit: &str) -> Result<OcrPageDimensions, Error> {
-    let scale = if unit == "inch" {
-        AZURE_DI_DEFAULT_DPI as f64
-    } else {
-        1.0
-    };
+fn convert_dimensions(
+    width: f64,
+    height: f64,
+    unit: &str,
+    dpi: i64,
+) -> Result<OcrPageDimensions, Error> {
+    let scale = if unit == "inch" { dpi as f64 } else { 1.0 };
     Ok(OcrPageDimensions {
         width: Some(pixel_dimension(width, scale, "page.width")?),
         height: Some(pixel_dimension(height, scale, "page.height")?),
-        dpi: Some(AZURE_DI_DEFAULT_DPI),
+        dpi: Some(dpi),
     })
 }
 
@@ -440,7 +448,7 @@ async fn read_operation_response(
     hooks: &dyn CallHooks<Error>,
 ) -> Result<DecodedOcrResponse<AzureDocumentIntelligenceOperation>, Error> {
     if response.status() != reqwest::StatusCode::ACCEPTED {
-        let bytes = crate::custom_httpx::llm_http_handler::read_response_bytes(
+        let bytes = crate::base_llm::ocr::handler::read_response_bytes(
             response,
             connection.max_response_bytes,
         )
@@ -462,11 +470,9 @@ async fn read_operation_response(
     {
         return Err(Error::PollOrigin);
     }
-    let bytes = crate::custom_httpx::llm_http_handler::read_response_bytes(
-        response,
-        connection.max_response_bytes,
-    )
-    .await?;
+    let bytes =
+        crate::base_llm::ocr::handler::read_response_bytes(response, connection.max_response_bytes)
+            .await?;
     hooks.response_received(&bytes).await?;
     poll_operation(http_client, operation, headers, connection, native, hooks).await
 }
@@ -480,7 +486,7 @@ async fn poll_operation(
     hooks: &dyn CallHooks<Error>,
 ) -> Result<DecodedOcrResponse<AzureDocumentIntelligenceOperation>, Error> {
     let deadline = Instant::now()
-        .checked_add(connection.poll_timeout)
+        .checked_add(connection.settings.poll_timeout)
         .ok_or(Error::PollTimeout)?;
 
     loop {
@@ -491,21 +497,19 @@ async fn poll_operation(
         let builder = http_client
             .get(url.clone())
             .timeout(remaining.min(connection.timeout));
-        let builder = crate::custom_httpx::http_handler::with_headers(
+        let builder = litellm_http::request::with_headers(
             builder,
             headers,
-            crate::custom_httpx::http_handler::HeaderPolicy::Only(&[
+            litellm_http::request::HeaderPolicy::Only(&[
                 AZURE_DI_SUBSCRIPTION_HEADER,
                 "authorization",
             ]),
         );
-        let response = tokio::time::timeout_at(
-            deadline,
-            crate::custom_httpx::http_handler::http_request(builder),
-        )
-        .await
-        .map_err(|_| Error::PollTimeout)?
-        .map_err(crate::custom_httpx::transport::Error::from)?;
+        let response =
+            tokio::time::timeout_at(deadline, litellm_http::request::http_request(builder))
+                .await
+                .map_err(|_| Error::PollTimeout)?
+                .map_err(litellm_http::transport::Error::from)?;
         let retry = response
             .headers()
             .get(reqwest::header::RETRY_AFTER)
@@ -551,13 +555,14 @@ impl AzureDocumentIntelligenceOcrConfig {
         endpoint: &str,
         model: &str,
         params: &DocumentIntelligenceParams,
+        api_version: &str,
     ) -> Result<String, Error> {
         let model = format!("{}:analyze", model_id(model)?);
         ApiUrl::parse(endpoint)
             .and_then(|url| url.complete_path(&["documentintelligence", "documentModels", &model]))
             .map(|url| {
                 url.append_query_pairs(
-                    [("api-version", AZURE_DI_API_VERSION)]
+                    [("api-version", api_version)]
                         .into_iter()
                         .chain(params.pages.iter().map(|pages| ("pages", pages.as_str())))
                         .chain(
@@ -580,8 +585,8 @@ impl AzureDocumentIntelligenceOcrConfig {
         config: &AzureAuthInputs,
         env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
     ) -> Result<Vec<(String, String)>, Error> {
-        if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization")
-            || crate::custom_httpx::http_handler::has_header(
+        if litellm_http::request::has_header(&connection.extra_headers, "authorization")
+            || litellm_http::request::has_header(
                 &connection.extra_headers,
                 AZURE_DI_SUBSCRIPTION_HEADER,
             )

litellm-rust/crates/llms/src/azure_ai/ocr/transformation.rs

@@ -7,12 +7,12 @@ use crate::{
     base_llm::ocr::{
         document::{inline_remote_document, validate_inline_document},
         error::Error,
+        handler::OcrClient,
         transformation::{
             BaseOcrConfig, LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrRequestContext,
-            OcrResponseFormat, PreparedOcrRequest, credential_env,
+            OcrResponseFormat, PreparedOcrRequest,
         },
     },
-    custom_httpx::llm_http_handler::OcrClient,
     mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest},
 };
 
@@ -50,15 +50,11 @@ impl BaseOcrConfig for AzureAiOcrConfig {
         request: &PreparedOcrRequest,
         _client: &OcrClient,
     ) -> Result<Self::Environment, Error> {
-        let config = AzureAuthInputs {
-            azure_ad_token_provider: request.azure_ad_token_provider.clone(),
-            ..AzureAuthInputs::from_sourced_optional_params(
-                &request.optional_params,
-                &request.input_sources,
-            )?
-        };
-        self.resolve_headers(&request.connection, &config, &credential_env)
-            .await
+        let config = crate::azure_ai::ocr::common_utils::azure_auth_inputs(request)?;
+        self.resolve_headers(&request.connection, &config, &|name: &str| {
+            request.connection.secret(name)
+        })
+        .await
     }
 
     fn get_complete_url(
@@ -67,7 +63,9 @@ impl BaseOcrConfig for AzureAiOcrConfig {
         _optional_params: &Self::OcrParams,
         _environment: &Self::Environment,
     ) -> Result<String, Error> {
-        self.build_ocr_url(request.connection.api_base.as_deref(), &credential_env)
+        self.build_ocr_url(request.connection.api_base.as_deref(), &|name: &str| {
+            request.connection.secret(name)
+        })
     }
 
     fn transform_ocr_request(
@@ -107,7 +105,7 @@ impl BaseOcrConfig for AzureAiOcrConfig {
     }
 
     fn validate_request_body(&self, body: &Value) -> Result<(), Error> {
-        validate_inline_document(&crate::custom_httpx::llm_http_handler::body_document(body)?)
+        validate_inline_document(&crate::base_llm::ocr::handler::body_document(body)?)
     }
 }
 
@@ -134,8 +132,7 @@ impl AzureAiOcrConfig {
         env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
     ) -> Result<Vec<(String, String)>, Error> {
         Self::resolve_api_base(connection.api_base.as_deref(), env_lookup)?;
-        if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization")
-        {
+        if litellm_http::request::has_header(&connection.extra_headers, "authorization") {
             if config.azure_ad_token_provider.is_some() {
                 super::common_utils::resolve_entra(config, env_lookup).await?;
             }

litellm-rust/crates/llms/src/base_llm/ocr/document.rs

@@ -1,18 +1,14 @@
 use base64::{Engine, engine::general_purpose::STANDARD};
 use data_url::{DataUrl, DataUrlError, forgiving_base64::DecodeError, mime::Mime};
+use litellm_http::{
+    media::{DownloadPolicy, Error as MediaError, MediaFetcher},
+    transport::Error as TransportError,
+};
 use reqwest::Url;
 
-use crate::{
-    base_llm::ocr::{
-        error::Error,
-        transformation::{
-            OCR_INLINE_MAX_BYTES, OCR_MAX_FETCH_REDIRECTS, OcrConnection, OcrDocument,
-        },
-    },
-    custom_httpx::{
-        media::{DownloadPolicy, Error as MediaError, MediaFetcher},
-        transport::Error as TransportError,
-    },
+use crate::base_llm::ocr::{
+    error::Error,
+    transformation::{OCR_INLINE_MAX_BYTES, OCR_MAX_FETCH_REDIRECTS, OcrConnection, OcrDocument},
 };
 
 pub struct InlineDocument<'a>(DataUrl<'a>);
@@ -72,7 +68,7 @@ pub async fn inline_remote_document(
             url,
             DownloadPolicy {
                 timeout: connection.timeout,
-                max_bytes: connection.max_download_bytes,
+                max_bytes: connection.settings.max_download_bytes,
                 max_redirects: OCR_MAX_FETCH_REDIRECTS,
             },
         )
@@ -196,10 +192,8 @@ mod tests {
             .redirect(reqwest::redirect::Policy::none())
             .build()
             .unwrap();
-        let client = crate::custom_httpx::llm_http_handler::OcrClient::for_test(
-            provider_http,
-            document_http,
-        );
+        let client =
+            crate::base_llm::ocr::handler::OcrClient::for_test(provider_http, document_http);
         let converted = inline_remote_document(
             client.document_fetcher(),
             OcrDocument::ImageUrl {

litellm-rust/crates/llms/src/base_llm/ocr/error.rs

@@ -95,11 +95,11 @@ pub enum Error {
     #[error(transparent)]
     Auth(#[from] litellm_auth::Error),
     #[error(transparent)]
-    Transport(#[from] crate::custom_httpx::transport::Error),
+    Transport(#[from] litellm_http::transport::Error),
     #[error(transparent)]
     Params(#[from] litellm_core_utils::params::Error),
     #[error(transparent)]
-    Headers(#[from] crate::custom_httpx::http_handler::HeaderError),
+    Headers(#[from] litellm_http::request::HeaderError),
 }
 
 impl From<litellm_host::machine::MachineFault> for Error {
@@ -125,9 +125,7 @@ impl Error {
     pub fn http_status_code(&self) -> Option<u16> {
         match self {
             Self::Provider { status, .. }
-            | Self::Transport(crate::custom_httpx::transport::Error::Http { status, .. }) => {
-                Some(*status)
-            }
+            | Self::Transport(litellm_http::transport::Error::Http { status, .. }) => Some(*status),
             error if error.is_request() => Some(400),
             _ => None,
         }

litellm-rust/crates/llms/src/base_llm/ocr/handler.rs

@@ -2,22 +2,21 @@ use bytes::{Bytes, BytesMut};
 use futures_util::future::BoxFuture;
 use litellm_auth_gcp::VertexAuth;
 use litellm_host::event::WireRequest;
-use litellm_http::{ClientVariant, HttpClientConfig, HttpClientPool};
+use litellm_http::{
+    ClientVariant, HttpClientConfig, HttpClientPool,
+    media::{MediaFetcher, UrlPolicy},
+    request::{HeaderPolicy, execute_http_request, with_headers},
+    transport,
+};
 use serde::{Serialize, de::DeserializeOwned};
 use serde_json::Value;
 
-use crate::{
-    base_llm::ocr::{
-        error::Error,
-        transformation::{
-            BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OcrDocument, OcrResponseContext,
-            PreparedOcrRequest, decode_request_value, decode_response,
-        },
-    },
-    custom_httpx::{
-        http_handler::{HeaderPolicy, execute_http_request, with_headers},
-        media::{MediaFetcher, UrlPolicy},
-        transport,
+use crate::base_llm::ocr::{
+    error::Error,
+    settings::{OcrSettings, Secrets},
+    transformation::{
+        BaseOcrConfig, DecodedOcrResponse, LiteLLMOcrResponse, OcrDocument, OcrResponseContext,
+        PreparedOcrRequest, decode_request_value, decode_response,
     },
 };
 
@@ -35,6 +34,8 @@ pub struct OcrClient {
     polling_http: reqwest::Client,
     document_fetcher: MediaFetcher,
     vertex_auth: VertexAuth,
+    settings: OcrSettings,
+    secrets: Secrets,
 }
 
 impl OcrClient {
@@ -43,12 +44,16 @@ impl OcrClient {
         config: &HttpClientConfig,
         url_policy: UrlPolicy,
         vertex_auth: VertexAuth,
+        settings: OcrSettings,
+        secrets: Secrets,
     ) -> Result<Self, litellm_http::Error> {
         Ok(Self {
             provider_http: pool.client(config, ClientVariant::Provider)?,
             polling_http: pool.client(config, ClientVariant::NoRedirect)?,
             document_fetcher: MediaFetcher::new(pool, config, url_policy)?,
             vertex_auth,
+            settings,
+            secrets,
         })
     }
 
@@ -68,6 +73,14 @@ impl OcrClient {
         &self.vertex_auth
     }
 
+    pub fn settings(&self) -> &OcrSettings {
+        &self.settings
+    }
+
+    pub fn secrets(&self) -> &Secrets {
+        &self.secrets
+    }
+
     #[cfg(any(test, feature = "test-support"))]
     pub fn for_test(provider_http: reqwest::Client, document_http: reqwest::Client) -> Self {
         Self {
@@ -78,8 +91,20 @@ impl OcrClient {
                 .expect("test polling client builds"),
             document_fetcher: MediaFetcher::for_test(document_http),
             vertex_auth: VertexAuth::default(),
+            settings: OcrSettings::default(),
+            secrets: std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment),
         }
     }
+
+    #[cfg(any(test, feature = "test-support"))]
+    pub fn with_settings(self, settings: OcrSettings) -> Self {
+        Self { settings, ..self }
+    }
+
+    #[cfg(any(test, feature = "test-support"))]
+    pub fn with_secrets(self, secrets: Secrets) -> Self {
+        Self { secrets, ..self }
+    }
 }
 
 /// Rust counterpart of `BaseLLMHTTPHandler.async_ocr`: prepare the provider request,

litellm-rust/crates/llms/src/base_llm/ocr/mod.rs

@@ -1,3 +1,5 @@
 pub mod document;
 pub mod error;
+pub mod handler;
+pub mod settings;
 pub mod transformation;

litellm-rust/crates/llms/src/base_llm/ocr/settings.rs

@@ -0,0 +1,147 @@
+use std::{sync::Arc, time::Duration};
+
+use litellm_core_utils::settings::Lookup;
+
+pub type Secrets = Arc<dyn Lookup + Send + Sync>;
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct OcrSettings {
+    pub request_timeout: Duration,
+    pub max_download_bytes: u64,
+    pub poll_timeout: Duration,
+    pub document_intelligence_api_version: String,
+    pub document_intelligence_dpi: i64,
+    pub vertex_project: Option<String>,
+    pub vertex_location: Option<String>,
+    pub enable_azure_ad_token_refresh: bool,
+}
+
+impl Default for OcrSettings {
+    fn default() -> Self {
+        Self {
+            request_timeout: Duration::from_secs(6000),
+            max_download_bytes: megabytes(50.0),
+            poll_timeout: Duration::from_secs(120),
+            document_intelligence_api_version: "2024-11-30".into(),
+            document_intelligence_dpi: 96,
+            vertex_project: None,
+            vertex_location: None,
+            enable_azure_ad_token_refresh: false,
+        }
+    }
+}
+
+impl OcrSettings {
+    pub fn from_environment(env: &impl Lookup) -> Self {
+        let defaults = Self::default();
+        Self {
+            request_timeout: env
+                .parsed::<f64>("REQUEST_TIMEOUT")
+                .and_then(|seconds| Duration::try_from_secs_f64(seconds).ok())
+                .unwrap_or(defaults.request_timeout),
+            max_download_bytes: env
+                .parsed::<f64>("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB")
+                .filter(|size| size.is_finite())
+                .map_or(defaults.max_download_bytes, megabytes),
+            poll_timeout: env
+                .parsed::<i64>("AZURE_OPERATION_POLLING_TIMEOUT")
+                .map_or(defaults.poll_timeout, |seconds| {
+                    Duration::from_secs(seconds.max(0).unsigned_abs())
+                }),
+            document_intelligence_api_version: env
+                .get("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION")
+                .unwrap_or(defaults.document_intelligence_api_version),
+            document_intelligence_dpi: env
+                .parsed("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI")
+                .unwrap_or(defaults.document_intelligence_dpi),
+            ..defaults
+        }
+    }
+}
+
+fn megabytes(size: f64) -> u64 {
+    (size * 1024.0 * 1024.0) as u64
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    fn env_of(values: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
+        move |name| {
+            values
+                .iter()
+                .find(|(key, _)| *key == name)
+                .map(|(_, value)| value.to_string())
+        }
+    }
+
+    #[test]
+    fn an_empty_environment_keeps_the_python_defaults() {
+        assert_eq!(
+            OcrSettings::from_environment(&env_of(&[])),
+            OcrSettings::default()
+        );
+    }
+
+    #[test]
+    fn every_setting_follows_its_environment_variable() {
+        let settings = OcrSettings::from_environment(&env_of(&[
+            ("REQUEST_TIMEOUT", "30.5"),
+            ("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", "0.5"),
+            ("AZURE_OPERATION_POLLING_TIMEOUT", " 600 "),
+            ("AZURE_DOCUMENT_INTELLIGENCE_API_VERSION", "2025-01-01"),
+            ("AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI", "72"),
+        ]));
+        assert_eq!(
+            settings,
+            OcrSettings {
+                request_timeout: Duration::from_millis(30_500),
+                max_download_bytes: 512 * 1024,
+                poll_timeout: Duration::from_secs(600),
+                document_intelligence_api_version: "2025-01-01".into(),
+                document_intelligence_dpi: 72,
+                ..OcrSettings::default()
+            }
+        );
+    }
+
+    #[rstest]
+    #[case::zero_disables_downloads("0", 0)]
+    #[case::negative_rejects_every_download("-1", 0)]
+    #[case::fraction_truncates_like_int("0.0000001", 0)]
+    #[case::unparsable_keeps_the_default("big", 50 * 1024 * 1024)]
+    fn download_size_converts_megabytes_like_python(
+        #[case] value: &'static str,
+        #[case] bytes: u64,
+    ) {
+        let env =
+            move |name: &str| (name == "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB").then(|| value.to_string());
+        assert_eq!(
+            OcrSettings::from_environment(&env).max_download_bytes,
+            bytes
+        );
+    }
+
+    #[test]
+    fn a_negative_polling_timeout_expires_immediately() {
+        let env =
+            |name: &str| (name == "AZURE_OPERATION_POLLING_TIMEOUT").then(|| "-5".to_string());
+        assert_eq!(
+            OcrSettings::from_environment(&env).poll_timeout,
+            Duration::ZERO
+        );
+    }
+
+    #[test]
+    fn an_empty_api_version_is_sent_as_is_like_python_str_of_getenv() {
+        let env =
+            |name: &str| (name == "AZURE_DOCUMENT_INTELLIGENCE_API_VERSION").then(String::new);
+        assert_eq!(
+            OcrSettings::from_environment(&env).document_intelligence_api_version,
+            ""
+        );
+    }
+}

litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs

@@ -1,9 +1,10 @@
-use std::{collections::BTreeMap, future::Future, time::Duration};
+use std::{collections::BTreeMap, future::Future, sync::Arc, time::Duration};
 
 use litellm_auth::{InputSource, SecretValue, Sourced, TokenProviderHandle};
 use litellm_core_utils::{
     call_arguments::CallArguments,
     serde_compat::{FiniteF64, LaxI64},
+    settings::ProcessEnvironment,
 };
 use serde::{
     Deserialize, Serialize,
@@ -12,19 +13,15 @@ use serde::{
 use serde_json::{Map, Value};
 use serde_with::serde_as;
 
-use crate::{
-    base_llm::ocr::error::Error,
-    custom_httpx::llm_http_handler::{
-        CallHooks, OcrClient, read_response_bytes, transform_request_body,
-    },
+use crate::base_llm::ocr::{
+    error::Error,
+    handler::{CallHooks, OcrClient, read_response_bytes, transform_request_body},
+    settings::{OcrSettings, Secrets},
 };
 
 pub const OCR_RESPONSE_MAX_BYTES: usize = 64 * 1024 * 1024;
-pub const OCR_HTTP_TIMEOUT_SECS: u64 = 600;
 pub const OCR_INLINE_MAX_BYTES: usize = 50 * 1024 * 1024;
-pub const OCR_DOWNLOAD_MAX_BYTES: u64 = 50 * 1024 * 1024;
 pub const OCR_MAX_FETCH_REDIRECTS: usize = 10;
-pub const OCR_POLL_TIMEOUT_SECS: u64 = 120;
 pub const OCR_POLL_RETRY_SECS: u64 = 2;
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@@ -116,10 +113,8 @@ impl OcrCredentialInputs {
 pub struct OcrTransportConfig {
     pub extra_headers: Vec<(String, String)>,
     pub extra_headers_source: InputSource,
-    pub timeout: Duration,
-    pub max_download_bytes: u64,
+    pub timeout: Option<Duration>,
     pub max_response_bytes: usize,
-    pub poll_timeout: Duration,
 }
 
 impl Default for OcrTransportConfig {
@@ -127,10 +122,8 @@ impl Default for OcrTransportConfig {
         Self {
             extra_headers: Vec::new(),
             extra_headers_source: InputSource::Deployment,
-            timeout: Duration::from_secs(OCR_HTTP_TIMEOUT_SECS),
-            max_download_bytes: OCR_DOWNLOAD_MAX_BYTES,
+            timeout: None,
             max_response_bytes: OCR_RESPONSE_MAX_BYTES,
-            poll_timeout: Duration::from_secs(OCR_POLL_TIMEOUT_SECS),
         }
     }
 }
@@ -145,7 +138,7 @@ impl OcrTransportConfig {
         Self {
             extra_headers,
             extra_headers_source,
-            timeout: timeout.unwrap_or(self.timeout),
+            timeout: timeout.or(self.timeout),
             ..self
         }
     }
@@ -166,13 +159,18 @@ pub struct OcrConnection {
     pub extra_headers: Vec<(String, String)>,
     pub extra_headers_source: InputSource,
     pub timeout: Duration,
-    pub max_download_bytes: u64,
     pub max_response_bytes: usize,
-    pub poll_timeout: Duration,
+    pub settings: OcrSettings,
+    pub secrets: Secrets,
 }
 
 impl OcrConnection {
-    pub fn new(credentials: ResolvedOcrCredentials, transport: OcrTransportConfig) -> Self {
+    pub fn new(
+        credentials: ResolvedOcrCredentials,
+        transport: OcrTransportConfig,
+        settings: OcrSettings,
+        secrets: Secrets,
+    ) -> Self {
         let api_key_source = credentials
             .api_key
             .as_ref()
@@ -190,12 +188,19 @@ impl OcrConnection {
             api_base_source,
             extra_headers: transport.extra_headers,
             extra_headers_source: transport.extra_headers_source,
-            timeout: transport.timeout,
-            max_download_bytes: transport.max_download_bytes,
+            timeout: transport
+                .timeout
+                .filter(|timeout| !timeout.is_zero())
+                .unwrap_or(settings.request_timeout),
             max_response_bytes: transport.max_response_bytes,
-            poll_timeout: transport.poll_timeout,
+            settings,
+            secrets,
         }
     }
+
+    pub fn secret(&self, name: &str) -> Option<String> {
+        self.secrets.get(name)
+    }
 }
 
 impl Default for OcrConnection {
@@ -203,6 +208,8 @@ impl Default for OcrConnection {
         Self::new(
             ResolvedOcrCredentials::default(),
             OcrTransportConfig::default(),
+            OcrSettings::default(),
+            Arc::new(ProcessEnvironment),
         )
     }
 }
@@ -565,16 +572,38 @@ pub fn decode_and_normalize_response<T: DeserializeOwned>(
     })
 }
 
-pub fn credential_env(name: &str) -> Option<String> {
-    std::env::var(name).ok()
-}
-
 #[cfg(test)]
 mod tests {
     use serde_json::json;
 
     use super::*;
 
+    #[test]
+    fn connection_timeout_falls_back_to_the_request_timeout_setting_like_a_python_or() {
+        let settings = OcrSettings {
+            request_timeout: Duration::from_secs(42),
+            ..OcrSettings::default()
+        };
+        let timeout = |call: Option<Duration>| {
+            OcrConnection::new(
+                ResolvedOcrCredentials::default(),
+                OcrTransportConfig {
+                    timeout: call,
+                    ..OcrTransportConfig::default()
+                },
+                settings.clone(),
+                Arc::new(ProcessEnvironment),
+            )
+            .timeout
+        };
+        assert_eq!(timeout(None), Duration::from_secs(42));
+        assert_eq!(timeout(Some(Duration::ZERO)), Duration::from_secs(42));
+        assert_eq!(
+            timeout(Some(Duration::from_secs(5))),
+            Duration::from_secs(5)
+        );
+    }
+
     #[test]
     fn normalized_response_rejects_invalid_shared_fields() {
         for fields in [

litellm-rust/crates/llms/src/cohere/ocr/transformation.rs

@@ -7,17 +7,15 @@ use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value};
 use serde_with::serde_as;
 
-use crate::{
-    base_llm::ocr::{
-        document::InlineDocument,
-        error::Error,
-        transformation::{
-            BaseOcrConfig, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES, OcrConnection, OcrDocument,
-            OcrPage, OcrPageImage, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
-            credential_env, decode_and_normalize_response, decode_response_value,
-        },
+use crate::base_llm::ocr::{
+    document::InlineDocument,
+    error::Error,
+    handler::OcrClient,
+    transformation::{
+        BaseOcrConfig, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES, OcrConnection, OcrDocument,
+        OcrPage, OcrPageImage, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
+        decode_and_normalize_response, decode_response_value,
     },
-    custom_httpx::llm_http_handler::OcrClient,
 };
 
 const COHERE_PARSE_API_BASE: &str = "https://api.cohere.com";
@@ -124,7 +122,9 @@ impl BaseOcrConfig for CohereParseConfig {
         request: &PreparedOcrRequest,
         _client: &OcrClient,
     ) -> Result<Self::Environment, Error> {
-        self.resolve_headers(&request.connection, &credential_env)
+        self.resolve_headers(&request.connection, &|name: &str| {
+            request.connection.secret(name)
+        })
     }
 
     fn get_complete_url(
@@ -163,7 +163,7 @@ impl BaseOcrConfig for CohereParseConfig {
     }
 
     fn validate_request_body(&self, body: &Value) -> Result<(), Error> {
-        validate_document(&crate::custom_httpx::llm_http_handler::body_document(body)?)
+        validate_document(&crate::base_llm::ocr::handler::body_document(body)?)
     }
 }
 
@@ -173,8 +173,7 @@ impl CohereParseConfig {
         connection: &OcrConnection,
         env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
     ) -> Result<Vec<(String, String)>, Error> {
-        if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization")
-        {
+        if litellm_http::request::has_header(&connection.extra_headers, "authorization") {
             return Ok(connection.extra_headers.clone());
         }
         let key = connection

litellm-rust/crates/llms/src/custom_httpx/mod.rs

@@ -1,4 +0,0 @@
-pub mod http_handler;
-pub mod llm_http_handler;
-pub mod media;
-pub mod transport;

litellm-rust/crates/llms/src/lib.rs

@@ -3,7 +3,6 @@ pub mod azure_ai;
 pub mod base_llm;
 pub mod bedrock;
 pub mod cohere;
-pub mod custom_httpx;
 pub mod mistral;
 pub mod openai;
 pub mod reducto;

litellm-rust/crates/llms/src/mistral/ocr/transformation.rs

@@ -2,16 +2,13 @@ use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, ur
 use serde::{Deserialize, Serialize};
 use serde_json::Value;
 
-use crate::{
-    base_llm::ocr::{
-        error::Error,
-        transformation::{
-            BaseOcrConfig, LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrPage,
-            OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest, credential_env,
-            decode_and_normalize_response,
-        },
+use crate::base_llm::ocr::{
+    error::Error,
+    handler::OcrClient,
+    transformation::{
+        BaseOcrConfig, LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrPage, OcrResponseFormat,
+        OcrUsageInfo, PreparedOcrRequest, decode_and_normalize_response,
     },
-    custom_httpx::llm_http_handler::OcrClient,
 };
 
 const MISTRAL_OCR_API_BASE: &str = "https://api.mistral.ai/v1";
@@ -87,7 +84,9 @@ impl BaseOcrConfig for MistralOcrConfig {
         request: &PreparedOcrRequest,
         _client: &OcrClient,
     ) -> Result<Self::Environment, Error> {
-        self.resolve_headers(&request.connection, &credential_env)
+        self.resolve_headers(&request.connection, &|name: &str| {
+            request.connection.secret(name)
+        })
     }
 
     fn get_complete_url(
@@ -129,8 +128,7 @@ impl MistralOcrConfig {
         connection: &OcrConnection,
         env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
     ) -> Result<Vec<(String, String)>, Error> {
-        if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization")
-        {
+        if litellm_http::request::has_header(&connection.extra_headers, "authorization") {
             return Ok(connection.extra_headers.clone());
         }
         let api_key = connection

litellm-rust/crates/llms/src/reducto/ocr/transformation.rs

@@ -8,18 +8,14 @@ use litellm_core_utils::{
 use serde::{Deserialize, Deserializer, Serialize};
 use serde_json::{Map, Value, json};
 
-use crate::{
-    base_llm::ocr::{
-        document::InlineDocument,
-        error::Error,
-        transformation::{
-            BaseOcrConfig, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES, OcrConnection, OcrDocument,
-            OcrPage, OcrRequestContext, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
-            credential_env, decode_and_normalize_response,
-        },
-    },
-    custom_httpx::llm_http_handler::{
-        CallHooks, OcrClient, build_http_request, guardrail_document,
+use crate::base_llm::ocr::{
+    document::InlineDocument,
+    error::Error,
+    handler::{CallHooks, OcrClient, build_http_request, guardrail_document},
+    transformation::{
+        BaseOcrConfig, LiteLLMOcrResponse, OCR_INLINE_MAX_BYTES, OcrConnection, OcrDocument,
+        OcrPage, OcrRequestContext, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
+        decode_and_normalize_response,
     },
 };
 
@@ -114,7 +110,9 @@ impl BaseOcrConfig for ReductoParseV3Config {
         request: &PreparedOcrRequest,
         _client: &OcrClient,
     ) -> Result<Self::Environment, Error> {
-        resolve_headers(&request.connection, &credential_env)
+        resolve_headers(&request.connection, &|name: &str| {
+            request.connection.secret(name)
+        })
     }
 
     fn get_complete_url(
@@ -437,7 +435,7 @@ fn resolve_headers(
     connection: &OcrConnection,
     env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
 ) -> Result<Vec<(String, String)>, Error> {
-    if crate::custom_httpx::http_handler::has_header(&connection.extra_headers, "authorization") {
+    if litellm_http::request::has_header(&connection.extra_headers, "authorization") {
         return Ok(connection.extra_headers.clone());
     }
     let api_key = connection
@@ -515,25 +513,21 @@ async fn upload_bytes_async(
         )?)
         .multipart(reqwest::multipart::Form::new().part("file", part))
         .timeout(connection.timeout);
-    let builder = crate::custom_httpx::http_handler::with_headers(
+    let builder = litellm_http::request::with_headers(
         builder,
         headers,
-        crate::custom_httpx::http_handler::HeaderPolicy::Except(&[
-            "content-type",
-            "content-length",
-        ]),
+        litellm_http::request::HeaderPolicy::Except(&["content-type", "content-length"]),
     );
-    let response = crate::custom_httpx::http_handler::http_request(builder)
+    let response = litellm_http::request::http_request(builder)
         .await
-        .map_err(crate::custom_httpx::transport::Error::from)?;
-    let uploaded =
-        crate::custom_httpx::llm_http_handler::read_json_response::<ReductoUploadResponse>(
-            response,
-            false,
-            connection.max_response_bytes,
-        )
-        .await?
-        .data;
+        .map_err(litellm_http::transport::Error::from)?;
+    let uploaded = crate::base_llm::ocr::handler::read_json_response::<ReductoUploadResponse>(
+        response,
+        false,
+        connection.max_response_bytes,
+    )
+    .await?
+    .data;
     let file_id = uploaded
         .file_id
         .as_deref()

litellm-rust/crates/llms/src/vertex_ai/ocr/common_utils.rs

@@ -1,6 +1,22 @@
 use litellm_auth::InputSource;
+use litellm_auth_gcp::VertexConfig;
 
-use crate::base_llm::ocr::{error::Error, transformation::OcrConnection};
+use crate::base_llm::ocr::{
+    error::Error,
+    transformation::{OcrConnection, PreparedOcrRequest},
+};
+
+pub(super) fn vertex_config(request: &PreparedOcrRequest) -> Result<VertexConfig, Error> {
+    let settings = &request.connection.settings;
+    Ok(VertexConfig::from_sourced_optional_params(
+        &request.optional_params,
+        &request.input_sources,
+    )?
+    .or_configured(
+        settings.vertex_project.as_deref(),
+        settings.vertex_location.as_deref(),
+    ))
+}
 
 pub(super) fn validate_destination(connection: &OcrConnection) -> Result<(), Error> {
     if connection.api_base.is_some() && connection.api_base_source == InputSource::Request {

litellm-rust/crates/llms/src/vertex_ai/ocr/deepseek_transformation.rs

@@ -1,19 +1,17 @@
-use litellm_auth_gcp::{self as vertex, VertexConfig};
+use litellm_auth_gcp as vertex;
 use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl};
 use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value};
 
-use super::transformation::VertexAiOcrConfig;
-use crate::{
-    base_llm::ocr::{
-        error::Error,
-        transformation::{
-            BaseOcrConfig, LiteLLMOcrResponse, OcrDocument, OcrPage, OcrPageDimensions,
-            OcrPageImage, OcrRequestContext, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
-            credential_env, decode_and_normalize_response, decode_response_value,
-        },
+use super::{common_utils::vertex_config, transformation::VertexAiOcrConfig};
+use crate::base_llm::ocr::{
+    error::Error,
+    handler::OcrClient,
+    transformation::{
+        BaseOcrConfig, LiteLLMOcrResponse, OcrDocument, OcrPage, OcrPageDimensions, OcrPageImage,
+        OcrRequestContext, OcrResponseFormat, OcrUsageInfo, PreparedOcrRequest,
+        decode_and_normalize_response, decode_response_value,
     },
-    custom_httpx::llm_http_handler::OcrClient,
 };
 
 const DEFAULT_API_BASE: &str = "https://aiplatform.googleapis.com";
@@ -124,12 +122,10 @@ impl BaseOcrConfig for VertexAIDeepSeekOCRConfig {
         _params: &Self::OcrParams,
         environment: &Self::Environment,
     ) -> Result<String, Error> {
-        let config = VertexConfig::from_sourced_optional_params(
-            &request.optional_params,
-            &request.input_sources,
-        )?;
-        let location = vertex::get_vertex_ai_location(&config, &credential_env)
-            .unwrap_or_else(|| DEFAULT_LOCATION.to_string());
+        let config = vertex_config(request)?;
+        let location =
+            vertex::get_vertex_ai_location(&config, &|name: &str| request.connection.secret(name))
+                .unwrap_or_else(|| DEFAULT_LOCATION.to_string());
         self.get_complete_url(
             request.connection.api_base.as_deref(),
             &environment.project_id,

litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs

@@ -2,17 +2,17 @@ use litellm_auth_gcp::{self as vertex, VertexConfig};
 use litellm_core_utils::{call_arguments::CallArguments, params::OpaqueParams, url_utils::ApiUrl};
 use serde_json::Value;
 
-use super::common_utils::validate_destination;
+use super::common_utils::{validate_destination, vertex_config};
 use crate::{
     base_llm::ocr::{
         document::{inline_remote_document, validate_inline_document},
         error::Error,
+        handler::OcrClient,
         transformation::{
             BaseOcrConfig, LiteLLMOcrResponse, OcrConnection, OcrDocument, OcrEnvironment,
-            OcrRequestContext, OcrResponseFormat, PreparedOcrRequest, credential_env,
+            OcrRequestContext, OcrResponseFormat, PreparedOcrRequest,
         },
     },
-    custom_httpx::llm_http_handler::OcrClient,
     mistral::ocr::transformation::{MistralOcrConfig, MistralOcrRequest},
 };
 
@@ -47,10 +47,7 @@ impl BaseOcrConfig for VertexAiOcrConfig {
         request: &PreparedOcrRequest,
         client: &OcrClient,
     ) -> Result<Self::Environment, Error> {
-        let config = VertexConfig::from_sourced_optional_params(
-            &request.optional_params,
-            &request.input_sources,
-        )?;
+        let config = vertex_config(request)?;
         self.resolve_environment(&request.connection, &config, client)
             .await
     }
@@ -61,12 +58,10 @@ impl BaseOcrConfig for VertexAiOcrConfig {
         _optional_params: &Self::OcrParams,
         environment: &Self::Environment,
     ) -> Result<String, Error> {
-        let config = VertexConfig::from_sourced_optional_params(
-            &request.optional_params,
-            &request.input_sources,
-        )?;
-        let location = vertex::get_vertex_ai_location(&config, &credential_env)
-            .unwrap_or_else(|| DEFAULT_LOCATION.to_string());
+        let config = vertex_config(request)?;
+        let location =
+            vertex::get_vertex_ai_location(&config, &|name: &str| request.connection.secret(name))
+                .unwrap_or_else(|| DEFAULT_LOCATION.to_string());
         self.build_ocr_url(
             request.connection.api_base.as_deref(),
             &environment.project_id,
@@ -112,7 +107,7 @@ impl BaseOcrConfig for VertexAiOcrConfig {
     }
 
     fn validate_request_body(&self, body: &Value) -> Result<(), Error> {
-        validate_inline_document(&crate::custom_httpx::llm_http_handler::body_document(body)?)
+        validate_inline_document(&crate::base_llm::ocr::handler::body_document(body)?)
     }
 }
 
@@ -139,7 +134,7 @@ impl VertexAiOcrConfig {
                     .as_ref()
                     .map(litellm_auth::SecretValue::expose),
                 config,
-                &credential_env,
+                &|name: &str| connection.secret(name),
             )
             .await
             .map_err(Error::from)

litellm-rust/crates/python-bridge/Cargo.toml

@@ -20,6 +20,7 @@ bytes.workspace = true
 litellm-auth.workspace = true
 litellm-callbacks-legacy.workspace = true
 litellm-core.workspace = true
+litellm-core-utils.workspace = true
 litellm-auth-gcp.workspace = true
 litellm-http.workspace = true
 litellm-llms.workspace = true

litellm-rust/crates/python-bridge/python_settings.json

@@ -14,5 +14,13 @@
   "url_policy": [
     "user_url_validation",
     "user_url_allowed_hosts"
+  ],
+  "provider_defaults": [
+    "vertex_project",
+    "vertex_location",
+    "enable_azure_ad_token_refresh"
+  ],
+  "secret_manager": [
+    "readable"
   ]
 }

litellm-rust/crates/python-bridge/src/errors.rs

@@ -1,7 +1,6 @@
 use litellm_core::{Error, audio_transcription, chat_completions, messages, responses};
-use litellm_llms::{
-    base_llm::ocr::error::Error as OcrError, custom_httpx::transport::Error as TransportError,
-};
+use litellm_http::transport::Error as TransportError;
+use litellm_llms::base_llm::ocr::error::Error as OcrError;
 use pyo3::{
     exceptions::{PyRuntimeError, PyValueError},
     prelude::*,

litellm-rust/crates/python-bridge/src/http.rs

@@ -4,11 +4,12 @@ use std::{
     sync::{Arc, LazyLock, Mutex, PoisonError},
 };
 
+use litellm_core_utils::settings::ProcessEnvironment;
 use litellm_http::{
     HttpClientConfig, HttpClientPool, HttpSettings, HttpSettingsLayer, Resolution, SslVerify,
     Unsupported,
+    media::{PublicDnsResolver, UrlPolicy},
 };
-use litellm_llms::custom_httpx::media::{PublicDnsResolver, UrlPolicy};
 use pyo3::{prelude::*, types::PyDict};
 
 use crate::{errors::RustBridgeDeclined, python_settings::PythonSettings};
@@ -29,7 +30,7 @@ pub(crate) fn call_config(
 ) -> PyResult<HttpClientConfig> {
     let settings = HttpSettings::from_layers([
         for_call(call_ssl_verify(kwargs)?, asynchronous),
-        HttpSettingsLayer::from_environment(&|name| std::env::var(name).ok()),
+        HttpSettingsLayer::from_environment(&ProcessEnvironment),
         configured(&PythonSettings::Http.read(py)?)?,
     ])
     .without_missing_files(&|path: &Path| path.exists());
@@ -232,7 +233,7 @@ user_agent='litellm/9.9.9',
         Python::initialize();
         Python::attach(|py| {
             let settings = HttpSettings::from_layers([
-                HttpSettingsLayer::from_environment(&|name| {
+                HttpSettingsLayer::from_environment(&|name: &str| {
                     (name == "LITELLM_USER_AGENT").then(|| "operator/1".to_string())
                 }),
                 configured(&python_settings(py, "")).unwrap(),

litellm-rust/crates/python-bridge/src/python_settings.rs

@@ -6,16 +6,25 @@ const MODULE: &str = "litellm.rust_bridge.settings";
 pub(crate) enum PythonSettings {
     Http,
     UrlPolicy,
+    ProviderDefaults,
+    SecretManager,
 }
 
 impl PythonSettings {
     #[cfg(test)]
-    pub(crate) const ALL: [Self; 2] = [Self::Http, Self::UrlPolicy];
+    pub(crate) const ALL: [Self; 4] = [
+        Self::Http,
+        Self::UrlPolicy,
+        Self::ProviderDefaults,
+        Self::SecretManager,
+    ];
 
     pub(crate) fn name(self) -> &'static str {
         match self {
             Self::Http => "http_settings",
             Self::UrlPolicy => "url_policy",
+            Self::ProviderDefaults => "provider_defaults",
+            Self::SecretManager => "secret_manager",
         }
     }
 

litellm-rust/crates/python-bridge/src/routes/messages/host.rs

@@ -4,7 +4,7 @@ use litellm_core::messages::{
     route::{Messages, MessagesCall, MessagesOp, MessagesOpResult, MessagesOutput},
 };
 use litellm_host_python::{InvokeError, RouteHost, from_py, lookup, to_py};
-use litellm_llms::custom_httpx::transport::Error as TransportError;
+use litellm_http::transport::Error as TransportError;
 use pyo3::{
     exceptions::{PyException, PyValueError},
     gc::{PyTraverseError, PyVisit},

litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs

@@ -15,10 +15,9 @@ pub(super) fn to_pyerr(error: Error) -> PyErr {
                 body,
                 headers,
             } => upstream_error(py, status, body, headers)?,
-            Error::Transport(litellm_llms::custom_httpx::transport::Error::Http {
-                status,
-                body,
-            }) => upstream_error(py, status, body, Vec::new())?,
+            Error::Transport(litellm_http::transport::Error::Http { status, body }) => {
+                upstream_error(py, status, body, Vec::new())?
+            }
             Error::RequestFormat => {
                 let error = core_error_to_pyerr(Error::RequestFormat.into());
                 error

litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs

@@ -3,19 +3,23 @@ mod errors;
 mod host;
 mod project;
 
-use std::sync::LazyLock;
+use std::sync::{Arc, LazyLock};
 
 use host::OcrRouteHost;
 use litellm_auth_gcp::VertexAuth;
 use litellm_callbacks_legacy::{LegacySurface, PublicCall, run_legacy_call};
 use litellm_core::ocr::route::ocr_machine;
-use litellm_llms::custom_httpx::llm_http_handler::OcrClient;
+use litellm_core_utils::settings::ProcessEnvironment;
+use litellm_llms::base_llm::ocr::{
+    handler::OcrClient,
+    settings::{OcrSettings, Secrets},
+};
 use pyo3::{
     prelude::*,
     types::{PyDict, PyTuple},
 };
 
-use crate::{errors::RustBridgeDeclined, http};
+use crate::{errors::RustBridgeDeclined, http, python_settings::PythonSettings};
 
 const SURFACE: LegacySurface = LegacySurface {
     call_type: "ocr",
@@ -37,12 +41,15 @@ fn run_ocr(
     kwargs: Bound<'_, PyDict>,
     asynchronous: bool,
 ) -> PyResult<Py<PyAny>> {
+    let secrets = process_environment_secrets(&PythonSettings::SecretManager.read(py)?)?;
     let config = http::call_config(py, &kwargs, asynchronous)?;
     let client = OcrClient::new(
         http::pool(),
         &config,
         http::url_policy(py)?,
         VERTEX_AUTH.clone(),
+        ocr_settings(py)?,
+        secrets,
     )
     .map_err(|error| RustBridgeDeclined::new_err(error.to_string()))?;
     run_legacy_call(
@@ -55,6 +62,45 @@ fn run_ocr(
     )
 }
 
+#[derive(FromPyObject)]
+struct PythonSecretManager {
+    readable: bool,
+}
+
+fn process_environment_secrets(secret_manager: &Bound<'_, PyAny>) -> PyResult<Secrets> {
+    let manager: PythonSecretManager = secret_manager.extract()?;
+    if manager.readable {
+        return Err(RustBridgeDeclined::new_err(
+            "a readable secret manager is configured and the Rust route only reads the process environment",
+        ));
+    }
+    Ok(Arc::new(ProcessEnvironment))
+}
+
+#[derive(FromPyObject)]
+struct PythonProviderDefaults {
+    vertex_project: Option<String>,
+    vertex_location: Option<String>,
+    enable_azure_ad_token_refresh: Option<bool>,
+}
+
+fn ocr_settings(py: Python<'_>) -> PyResult<OcrSettings> {
+    let defaults: PythonProviderDefaults = PythonSettings::ProviderDefaults
+        .read(py)?
+        .extract()
+        .map_err(|error: PyErr| {
+            RustBridgeDeclined::new_err(format!(
+                "litellm provider defaults cannot be used by the Rust route: {error}"
+            ))
+        })?;
+    Ok(OcrSettings {
+        vertex_project: defaults.vertex_project,
+        vertex_location: defaults.vertex_location,
+        enable_azure_ad_token_refresh: defaults.enable_azure_ad_token_refresh == Some(true),
+        ..OcrSettings::from_environment(&ProcessEnvironment)
+    })
+}
+
 #[pyfunction]
 pub(crate) fn ocr(
     py: Python<'_>,
@@ -74,3 +120,47 @@ pub(crate) fn aocr(
 ) -> PyResult<Py<PyAny>> {
     run_ocr(py, request, args, kwargs, true)
 }
+
+#[cfg(test)]
+mod tests {
+    use pyo3::{prelude::*, types::PyDict};
+
+    use super::process_environment_secrets;
+    use crate::errors::RustBridgeDeclined;
+
+    fn secret_manager<'py>(py: Python<'py>, readable: bool) -> Bound<'py, PyAny> {
+        let locals = PyDict::new(py);
+        locals.set_item("readable", readable).unwrap();
+        py.run(
+            c"import types\nmanager = types.SimpleNamespace(readable=readable)",
+            Some(&locals),
+            Some(&locals),
+        )
+        .unwrap();
+        locals.get_item("manager").unwrap().unwrap()
+    }
+
+    #[test]
+    fn a_readable_secret_manager_sends_the_call_back_to_python() {
+        Python::initialize();
+        Python::attach(|py| {
+            let declined = process_environment_secrets(&secret_manager(py, true))
+                .err()
+                .expect("the Rust route declines");
+            assert!(declined.is_instance_of::<RustBridgeDeclined>(py));
+        });
+    }
+
+    #[test]
+    fn without_a_readable_secret_manager_secrets_are_the_process_environment() {
+        Python::initialize();
+        Python::attach(|py| {
+            let secrets = process_environment_secrets(&secret_manager(py, false)).unwrap();
+            assert_eq!(
+                secrets.get("LITELLM_RUST_BRIDGE_UNSET_VARIABLE_FOR_TEST"),
+                None
+            );
+            assert_eq!(secrets.get("PATH"), std::env::var("PATH").ok());
+        });
+    }
+}

litellm-rust/crates/python-bridge/src/routes/ocr/project.rs

@@ -592,7 +592,7 @@ kwargs = {
             );
             assert_eq!(
                 projected.transport.timeout,
-                std::time::Duration::from_secs(5)
+                Some(std::time::Duration::from_secs(5))
             );
         });
     }

litellm/__init__.py

@@ -701,6 +701,7 @@ github_copilot_models: Set = set()
 chatgpt_models: Set = set()
 minimax_models: Set = set()
 aws_polly_models: Set = set()
+transcribe_models: Set = set()
 gigachat_models: Set = set()
 llamagate_models: Set = set()
 reducto_models: Set = set()
@@ -980,6 +981,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None:
             minimax_models.add(key)
         elif value.get("litellm_provider") == "aws_polly":
             aws_polly_models.add(key)
+        elif value.get("litellm_provider") == "transcribe":
+            transcribe_models.add(key)
         elif value.get("litellm_provider") == "gigachat":
             gigachat_models.add(key)
         elif value.get("litellm_provider") == "llamagate":
@@ -1227,6 +1230,7 @@ def _build_models_by_provider() -> dict:
         "chatgpt": chatgpt_models,
         "minimax": minimax_models,
         "aws_polly": aws_polly_models,
+        "transcribe": transcribe_models,
         "gigachat": gigachat_models,
         "llamagate": llamagate_models,
         "reducto": reducto_models,

litellm/batches/batch_utils.py

@@ -9,6 +9,7 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
 from litellm.litellm_core_utils.llm_cost_calc.utils import parse_prompt_tokens_details
+from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
 from litellm.llms.bedrock.batches.transformation import titan_embedding_usage_from_batch_output
 from litellm.llms.vertex_ai.batches.transformation import vertex_prompt_tokens_details
 from litellm.types.llms.openai import Batch
@@ -52,7 +53,7 @@ def batch_cost_is_final(batch: Batch) -> bool:
 
 async def calculate_batch_cost_and_usage(
     file_content_dictionary: list[dict],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
 ) -> BatchCostUsageResult:
@@ -82,7 +83,7 @@ async def calculate_batch_cost_and_usage(
 
 async def _handle_completed_batch(
     batch: Batch,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
     model_name: str | None = None,
     litellm_params: dict | None = None,
     model_info: ModelInfo | None = None,
@@ -168,7 +169,7 @@ class _BatchOutputLineStats:
 
 def _classify_output_line_stats(
     entries: Iterable[dict],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
 ) -> Iterator[_BatchOutputLineStats | _LineOutcome]:
@@ -187,7 +188,7 @@ def _classify_output_line_stats(
 
 def _safe_output_line_stats(
     entry: Mapping[str, object],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
 ) -> _BatchOutputLineStats | None:
@@ -209,7 +210,7 @@ def _safe_output_line_stats(
 
 def _compute_output_line_stats(
     entry: Mapping[str, object],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     model_info: ModelInfo | None,
 ) -> _BatchOutputLineStats:
@@ -220,6 +221,7 @@ def _compute_output_line_stats(
     response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
     completion_details: Final = usage.completion_tokens_details
     line_prompt_cost, line_completion_cost = _output_line_cost(
+        response_body=response_body,
         usage=usage,
         custom_llm_provider=custom_llm_provider,
         model_name=model_name,
@@ -239,19 +241,36 @@ def _compute_output_line_stats(
     )
 
 
+def _ocr_usage_info_from_response_body(response_body: Mapping[str, object]) -> OCRUsageInfo | None:
+    """OCR results report ``usage_info`` (pages) instead of ``usage`` (tokens); None for non-OCR lines."""
+    raw_usage_info: Final = response_body.get("usage_info")
+    if not isinstance(raw_usage_info, Mapping):
+        return None
+    return OCRUsageInfo.model_validate(raw_usage_info)
+
+
 def _output_line_cost(
+    response_body: Mapping[str, object],
     usage: Usage,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None,
     response_model: str | None,
     model_info: ModelInfo | None,
 ) -> tuple[float, float]:
     """(prompt_cost, completion_cost) for one output line, priced at batch rates."""
-    from litellm.cost_calculator import batch_cost_calculator
+    from litellm.cost_calculator import batch_cost_calculator, ocr_batch_cost
 
     cost_model: Final = (
         model_name if custom_llm_provider == "bedrock" and model_name else response_model or model_name or ""
     )
+    ocr_usage: Final = _ocr_usage_info_from_response_body(response_body)
+    if ocr_usage is not None:
+        return ocr_batch_cost(
+            model=cost_model,
+            custom_llm_provider=custom_llm_provider,
+            usage_info=ocr_usage,
+            model_info=model_info,
+        )
     return batch_cost_calculator(
         usage=usage,
         model=cost_model,
@@ -262,7 +281,7 @@ def _output_line_cost(
 
 def _aggregate_batch_cost_usage_models(
     entries: Iterable[dict],
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock"],
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
     model_name: str | None = None,
     model_info: ModelInfo | None = None,
 ) -> BatchCostUsageResult:
@@ -430,7 +449,7 @@ def _provider_output_file_id(output_file_id: str) -> str:
 
 async def _fetch_batch_managed_file_content(
     file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"] = "openai",
     litellm_params: dict | None = None,
 ) -> bytes:
     """
@@ -460,7 +479,7 @@ async def _fetch_batch_managed_file_content(
 
 async def _fetch_batch_output_file_content(
     batch: Batch,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"] = "openai",
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"] = "openai",
     litellm_params: dict | None = None,
 ) -> bytes:
     """
@@ -482,7 +501,7 @@ async def _fetch_batch_output_file_content(
 
 async def count_error_file_failed_requests(
     batch: Batch,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic"],
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
     litellm_params: dict | None,
 ) -> int:
     """Count failed requests reported only in the batch's separate error file.

litellm/batches/main.py

@@ -105,9 +105,11 @@ def _resolve_timeout(
 @client
 async def acreate_batch(
     completion_window: Literal["24h"],
-    endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"],
+    endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"],
     input_file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy"] = "openai",
+    custom_llm_provider: Literal[
+        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral"
+    ] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -155,9 +157,11 @@ async def acreate_batch(
 @client
 def create_batch(
     completion_window: Literal["24h"],
-    endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses"],
+    endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"],
     input_file_id: str,
-    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy"] = "openai",
+    custom_llm_provider: Literal[
+        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral"
+    ] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
     extra_body: dict[str, str] | None = None,
@@ -341,7 +345,7 @@ def create_batch(
 async def aretrieve_batch(
     batch_id: str,
     custom_llm_provider: Literal[
-        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic"
+        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral"
     ] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,
@@ -389,7 +393,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
     _retrieve_batch_request: RetrieveBatchRequest,
     _is_async: bool,
     custom_llm_provider: Literal[
-        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic"
+        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral"
     ] = "openai",
     logging_obj: LiteLLMLoggingObj | None = None,
 ):
@@ -497,7 +501,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
             message=(
                 f"LiteLLM doesn't support custom_llm_provider={custom_llm_provider} for 'retrieve_batch' without a `model` kwarg. "
                 "Supported via this path: 'openai', 'azure', 'vertex_ai', 'anthropic'. "
-                "'bedrock' is supported but requires `model` to be passed so the provider config can be loaded."
+                "'bedrock' and 'mistral' are supported but require `model` to be passed so the provider config can be loaded."
             ),
             model="n/a",
             llm_provider=custom_llm_provider,
@@ -514,7 +518,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
 def retrieve_batch(
     batch_id: str,
     custom_llm_provider: Literal[
-        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic"
+        "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral"
     ] = "openai",
     metadata: dict[str, str] | None = None,
     extra_headers: dict[str, str] | None = None,

litellm/cost_calculator.py

@@ -141,6 +141,7 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import (
         Logging as LitellmLoggingObject,
     )
+    from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
 else:
     LitellmLoggingObject = Any
 
@@ -2114,6 +2115,87 @@ def ocr_cost(
     return ocr_pages_cost + annotation_pages_cost, 0.0
 
 
+_OCR_BATCH_PAGE_RATE_KEYS: Final = ("ocr_cost_per_page_batches", "ocr_cost_per_page")
+_OCR_BATCH_ANNOTATION_RATE_KEYS: Final = ("annotation_cost_per_page_batches", "annotation_cost_per_page")
+
+
+def ocr_batch_cost(
+    model: str,
+    custom_llm_provider: str | None,
+    usage_info: "OCRUsageInfo",
+    model_info: ModelInfo | None = None,
+) -> tuple[float, float]:
+    """Per-page cost of one OCR result inside a batch output file.
+
+    Batch OCR is billed per page at the ``*_batches`` rate, falling back to the
+    synchronous per-page rate when a model has no batch price recorded, the same
+    fallback ``batch_cost_calculator`` applies to per-token batch pricing. Each
+    per-page family (OCR pages, annotation pages) belongs to the deployment's
+    ``model_info`` when it prices that family at either rate and to the published
+    cost map otherwise, so a deployment overriding one family keeps the model's
+    published rate for the other, and the cost map is only consulted for a family
+    the deployment leaves out. Returns ``(prompt_cost, completion_cost)`` with the
+    whole cost in the first slot, like ``ocr_cost``.
+    """
+    pages_processed: Final = usage_info.pages_processed or 0
+    annotation_pages: Final = usage_info.pages_processed_annotation or 0
+    deployment_page_rate: Final = _first_price(model_info, *_OCR_BATCH_PAGE_RATE_KEYS)
+    deployment_annotation_rate: Final = _first_price(model_info, *_OCR_BATCH_ANNOTATION_RATE_KEYS)
+    needs_published_pricing: Final = (pages_processed > 0 and deployment_page_rate is None) or (
+        annotation_pages > 0 and deployment_annotation_rate is None
+    )
+    published: Final = (
+        _lookup_model_info_or_none(model=model, custom_llm_provider=custom_llm_provider)
+        if needs_published_pricing
+        else None
+    )
+    if needs_published_pricing and published is None:
+        verbose_logger.warning(
+            "OCR batch cost: model=%s custom_llm_provider=%s has no pricing entry; "
+            "billing only the per-page families the deployment prices.",
+            _single_log_line(model),
+            _single_log_line(custom_llm_provider),
+        )
+
+    page_rate: Final = (
+        deployment_page_rate
+        if deployment_page_rate is not None
+        else _first_price(published, *_OCR_BATCH_PAGE_RATE_KEYS)
+    )
+    annotation_rate: Final = (
+        deployment_annotation_rate
+        if deployment_annotation_rate is not None
+        else _first_price(published, *_OCR_BATCH_ANNOTATION_RATE_KEYS)
+    )
+    if page_rate is None and pages_processed > 0:
+        verbose_logger.warning(
+            "OCR batch cost: model=%s custom_llm_provider=%s reported pages_processed=%s but no "
+            "ocr_cost_per_page is configured; returning 0.0 cost for those pages.",
+            _single_log_line(model),
+            _single_log_line(custom_llm_provider),
+            pages_processed,
+        )
+    effective_annotation_rate: Final = annotation_rate if annotation_rate is not None else page_rate
+    return (page_rate or 0.0) * pages_processed + (effective_annotation_rate or 0.0) * annotation_pages, 0.0
+
+
+def _single_log_line(value: str | None) -> str:
+    return str(value).replace("\n", "").replace("\r", "")
+
+
+def _lookup_model_info_or_none(model: str, custom_llm_provider: str | None) -> ModelInfo | None:
+    try:
+        return litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
+    except Exception:  # noqa: BLE001  # get_model_info raises bare Exception for unmapped models; caller logs and bills 0.0
+        return None
+
+
+def _first_price(model_info: ModelInfo | None, *keys: str) -> float | None:
+    if model_info is None:
+        return None
+    return next((price for price in (model_info.get(k) for k in keys) if isinstance(price, (int, float))), None)
+
+
 def vector_store_search_cost(
     model: str | None,
     custom_llm_provider: str,

litellm/exceptions.py

@@ -787,6 +787,7 @@ class InternalServerError(openai.InternalServerError):
         super().__init__(
             self.message, response=self.response, body=body
         )  # Call the base class constructor with the parameters it needs
+        self.type = "internal_server_error"
 
     def __str__(self):
         _message = self.message

litellm/files/main.py

@@ -27,12 +27,13 @@ FileCreateProvider = Literal[
     "litellm_proxy",
     "manus",
     "anthropic",
+    "mistral",
 ]
 FileRetrieveProvider = Literal[
-    "openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic"
+    "openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic", "mistral"
 ]
-FileDeleteProvider = Literal["openai", "azure", "gemini", "bedrock", "litellm_proxy", "manus", "anthropic"]
-FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic"]
+FileDeleteProvider = Literal["openai", "azure", "gemini", "bedrock", "litellm_proxy", "manus", "anthropic", "mistral"]
+FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic", "mistral"]
 import litellm
 from litellm import get_secret_str
 from litellm.files.streaming import FileContentStreamingResponse

litellm/files/types.py

@@ -2,7 +2,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import Literal, NamedTuple
 
 FileContentProvider = Literal[
-    "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "manus"
+    "openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "manus", "mistral"
 ]
 
 

litellm/integrations/SlackAlerting/slack_alerting.py

@@ -647,10 +647,10 @@ class SlackAlerting(CustomBatchLogger):
                 event_message += f"Budget Crossed\n Total Budget:`{user_info.max_budget}`"
             elif percent_left <= SLACK_ALERTING_THRESHOLD_5_PERCENT:
                 event = "threshold_crossed"
-                event_message += "5% Threshold Crossed "
+                event_message += "5% or less of budget remaining"
             elif percent_left <= SLACK_ALERTING_THRESHOLD_15_PERCENT:
                 event = "threshold_crossed"
-                event_message += "15% Threshold Crossed"
+                event_message += "15% or less of budget remaining"
 
         return event, event_message
 

litellm/integrations/otel/logger.py

@@ -5,7 +5,7 @@ from collections.abc import Callable, Iterator, Mapping, Sequence
 from contextlib import contextmanager
 from datetime import datetime
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, cast
+from typing import TYPE_CHECKING, Final, cast
 
 from opentelemetry.context import Context, attach, get_current
 from opentelemetry.sdk._logs import LoggerProvider
@@ -21,6 +21,7 @@ from opentelemetry.trace import (
     use_span,
 )
 from opentelemetry.trace import TracerProvider as ApiTracerProvider
+from typing_extensions import TypedDict, Unpack
 
 import litellm
 from litellm._logging import verbose_logger
@@ -140,6 +141,10 @@ def _request_trace_links(context: Context | None) -> tuple[Link, ...] | None:
     return (Link(anchor),) if anchor.is_valid else None
 
 
+class _CustomLoggerOptions(TypedDict, total=False, extra_items=object):
+    pass
+
+
 class _LLMCallSpan:
     """The state carried from the ``pre_call`` boundary to span close.
 
@@ -179,7 +184,7 @@ class OpenTelemetryV2(CustomLogger):
         tracer_provider: TracerProvider | None = None,
         logger_provider: LoggerProvider | None = None,
         meter_provider: "MeterProvider | None" = None,
-        **kwargs: Any,
+        **kwargs: Unpack[_CustomLoggerOptions],
     ) -> None:
         super().__init__(**kwargs)
         self.config: OpenTelemetryV2Config = config or OpenTelemetryV2Config(**kwargs)

litellm/integrations/websearch_interception/handler.py

@@ -44,6 +44,7 @@ from litellm.types.integrations.custom_logger import (
 from litellm.types.integrations.websearch_interception import (
     AnthropicSearchQuery,
     AnthropicServerToolUseBlock,
+    RichWebSearchInput,
     SearchFailed,
     SearchOutcome,
     WebSearchInterceptionConfig,
@@ -1144,7 +1145,9 @@ class WebSearchInterceptionLogger(CustomLogger):
         """Execute litellm.asearch() and build a Responses API rerun patch."""
         search_tasks: Final = [
             (
-                self._execute_search(tool_call["input"]["query"], kwargs=kwargs)
+                self._execute_search(
+                    tool_call["input"]["query"], kwargs=kwargs, rich=self._rich_search_input(tool_call["input"])
+                )
                 if isinstance(tool_call.get("input"), dict) and tool_call["input"].get("query")
                 else self._create_empty_search_result()
             )
@@ -1362,7 +1365,9 @@ class WebSearchInterceptionLogger(CustomLogger):
             query = tool_call["input"].get("query")
             if query:
                 verbose_logger.debug("WebSearchInterception: Queuing search for query='%s'", query)
-                search_tasks.append(self._execute_search(query, kwargs=kwargs))
+                search_tasks.append(
+                    self._execute_search(query, kwargs=kwargs, rich=self._rich_search_input(tool_call["input"]))
+                )
             else:
                 verbose_logger.debug("WebSearchInterception: Tool call %s has no query", tool_call["id"])
                 # Add empty result for tools without query
@@ -1431,8 +1436,53 @@ class WebSearchInterceptionLogger(CustomLogger):
             return WebSearchTransformation.search_outcome(e)
         return WebSearchTransformation.search_outcome(result)
 
+    @staticmethod
+    def _rich_search_input(tool_input: object) -> RichWebSearchInput | None:
+        """
+        Extract the optional objective/search_queries pair from a tool input.
+
+        Returns None when the input carries neither, so callers can pass the
+        result straight through as ``_execute_search``'s ``rich`` argument.
+        """
+        if not isinstance(tool_input, Mapping):
+            return None
+        objective = tool_input.get("objective")
+        valid_objective = objective if isinstance(objective, str) and objective.strip() else None
+        raw_queries = tool_input.get("search_queries")
+        valid_queries: list[str] | None = None  # mutable-ok: matches litellm.asearch's list[str] query parameter
+        if isinstance(raw_queries, Sequence) and not isinstance(raw_queries, str):
+            queries = [q for q in raw_queries if isinstance(q, str) and q.strip()]
+            if queries:
+                # Providers cap multi-query requests (Parallel drops queries
+                # past the fifth); trim here so nothing is silently ignored.
+                valid_queries = queries[:5]
+        if valid_objective is not None and valid_queries is not None:
+            return {"objective": valid_objective, "search_queries": valid_queries}
+        if valid_objective is not None:
+            return {"objective": valid_objective}
+        if valid_queries is not None:
+            return {"search_queries": valid_queries}
+        return None
+
+    @staticmethod
+    def _provider_supports_rich_search(search_provider: str | None) -> bool:
+        """Whether the provider's search config accepts objective + multi-query input."""
+        if not search_provider:
+            return False
+        try:
+            from litellm.utils import ProviderConfigManager
+        except ImportError:
+            return False
+        # SearchProviders is a str enum, so an unknown provider string simply
+        # misses the config map and returns None rather than raising.
+        config = ProviderConfigManager.get_provider_search_config(search_provider)  # pyright: ignore[reportArgumentType] -- SearchProviders is a str enum, so the router's provider string hashes to the matching member; unknown strings miss the map and yield None
+        return config is not None and config.supports_rich_search_input()
+
     async def _execute_search(
-        self, query: str, kwargs: Mapping[str, object] | None = None
+        self,
+        query: str,
+        kwargs: Mapping[str, object] | None = None,
+        rich: RichWebSearchInput | None = None,
     ) -> tuple[str, SearchResponse | None]:
         """
         Execute a single web search using router's search tools.
@@ -1490,13 +1540,24 @@ class WebSearchInterceptionLogger(CustomLogger):
                 for key, value in search_litellm_params.items()
                 if key != "search_provider" and value is not None
             }
+            # Forward the model's richer shape (objective + keyword queries)
+            # only to providers whose search API takes it natively; everyone
+            # else keeps the single query string the model also provided.
+            query_arg: str | list[str] = query  # mutable-ok: litellm.asearch declares query as str | list[str]
+            if rich and self._provider_supports_rich_search(search_provider):
+                rich_queries = rich.get("search_queries")
+                if rich_queries:
+                    query_arg = rich_queries
+                rich_objective = rich.get("objective")
+                if rich_objective and "objective" not in search_kwargs:
+                    search_kwargs["objective"] = rich_objective
             result: Final = (
                 await litellm.asearch(
-                    query=query, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
+                    query=query_arg, search_provider=search_provider, **_NO_ASEARCH_NAMED, **search_kwargs
                 )
                 if search_metadata is None
                 else await litellm.asearch(
-                    query=query,
+                    query=query_arg,
                     search_provider=search_provider,
                     litellm_metadata=search_metadata,
                     **_NO_ASEARCH_NAMED,
@@ -1701,18 +1762,21 @@ class WebSearchInterceptionLogger(CustomLogger):
         for tool_call in tool_calls:
             # Handle both Anthropic-style input and OpenAI-style function.arguments
             query = None
+            tool_args: dict | None = None  # mutable-ok: the tool call's own arguments dict
             if "input" in tool_call and isinstance(tool_call["input"], dict):
-                query = tool_call["input"].get("query")
+                tool_args = tool_call["input"]
+                query = tool_args.get("query")
             elif "function" in tool_call:
                 func = tool_call["function"]
                 if isinstance(func, dict):
                     args = func.get("arguments", {})
                     if isinstance(args, dict):
+                        tool_args = args
                         query = args.get("query")
 
             if query:
                 verbose_logger.debug("WebSearchInterception: Queuing search for query='%s'", query)
-                search_tasks.append(self._execute_search(query, kwargs=kwargs))
+                search_tasks.append(self._execute_search(query, kwargs=kwargs, rich=self._rich_search_input(tool_args)))
             else:
                 verbose_logger.debug("WebSearchInterception: Tool call %s has no query", tool_call.get("id"))
                 # Add empty result for tools without query

litellm/integrations/websearch_interception/tools.py

@@ -11,6 +11,50 @@ from typing import Any, Final
 
 from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
 
+_WEB_SEARCH_TOOL_DESCRIPTION: Final = (
+    "Search the web for information. Use this when you need current "
+    "information or answers to questions that require up-to-date data."
+)
+
+
+def _web_search_input_schema() -> dict[str, object]:  # mutable-ok: plain-dict tool shape, as the get_* builders
+    """
+    JSON schema for the web search tool's input, shared by every tool format.
+
+    ``query`` stays required so providers and callers that only understand a
+    single query string keep working unchanged. ``objective`` and
+    ``search_queries`` are optional richer inputs; they are forwarded only to
+    search providers that support them (see
+    ``BaseSearchConfig.supports_rich_search_input``).
+    """
+    return {
+        "type": "object",
+        "properties": {
+            "query": {
+                "type": "string",
+                "description": "The search query to execute",
+            },
+            "objective": {
+                "type": "string",
+                "description": (
+                    "Natural-language description of the goal behind the "
+                    "search, including any source or freshness requirements."
+                ),
+            },
+            "search_queries": {
+                "type": "array",
+                "items": {"type": "string"},
+                "description": (
+                    "Two to five short keyword queries (3-6 words each) "
+                    "covering different angles of the objective, e.g. varying "
+                    "names, synonyms, or phrasings. Provide together with "
+                    "objective for the best results."
+                ),
+            },
+        },
+        "required": ["query"],
+    }
+
 
 def get_litellm_web_search_tool() -> dict[str, object]:
     """
@@ -33,20 +77,8 @@ def get_litellm_web_search_tool() -> dict[str, object]:
     """
     return {
         "name": LITELLM_WEB_SEARCH_TOOL_NAME,
-        "description": (
-            "Search the web for information. Use this when you need current "
-            "information or answers to questions that require up-to-date data."
-        ),
-        "input_schema": {
-            "type": "object",
-            "properties": {
-                "query": {
-                    "type": "string",
-                    "description": "The search query to execute",
-                }
-            },
-            "required": ["query"],
-        },
+        "description": _WEB_SEARCH_TOOL_DESCRIPTION,
+        "input_schema": _web_search_input_schema(),
     }
 
 
@@ -65,20 +97,8 @@ def get_litellm_web_search_tool_openai() -> dict[str, object]:
         "type": "function",
         "function": {
             "name": LITELLM_WEB_SEARCH_TOOL_NAME,
-            "description": (
-                "Search the web for information. Use this when you need current "
-                "information or answers to questions that require up-to-date data."
-            ),
-            "parameters": {
-                "type": "object",
-                "properties": {
-                    "query": {
-                        "type": "string",
-                        "description": "The search query to execute",
-                    }
-                },
-                "required": ["query"],
-            },
+            "description": _WEB_SEARCH_TOOL_DESCRIPTION,
+            "parameters": _web_search_input_schema(),
         },
     }
 
@@ -98,20 +118,8 @@ def get_litellm_web_search_tool_responses() -> dict[str, object]:
     return {
         "type": "function",
         "name": LITELLM_WEB_SEARCH_TOOL_NAME,
-        "description": (
-            "Search the web for information. Use this when you need current "
-            "information or answers to questions that require up-to-date data."
-        ),
-        "parameters": {
-            "type": "object",
-            "properties": {
-                "query": {
-                    "type": "string",
-                    "description": "The search query to execute",
-                }
-            },
-            "required": ["query"],
-        },
+        "description": _WEB_SEARCH_TOOL_DESCRIPTION,
+        "parameters": _web_search_input_schema(),
     }
 
 

litellm/litellm_core_utils/litellm_logging.py

@@ -371,6 +371,10 @@ _DEPLOYMENT_PRICING_KEYS: Final = (
     "output_cost_per_token",
     "input_cost_per_token_batches",
     "output_cost_per_token_batches",
+    "ocr_cost_per_page",
+    "ocr_cost_per_page_batches",
+    "annotation_cost_per_page",
+    "annotation_cost_per_page_batches",
 )
 
 
@@ -386,7 +390,9 @@ def deployment_pricing_model_info(model_id: str | None, deployment_model: str |
     the model's published rates instead of billing as zero. Ownership is per
     token direction: declaring either rate for a direction takes that whole
     direction, so a published batch rate can never displace a standard rate
-    the deployment configured itself.
+    the deployment configured itself. OCR per-page rates count as declared
+    pricing too; they pass through as registered and ``ocr_batch_cost`` layers
+    the published rate under each per-page family the deployment leaves out.
     """
     if model_id is None:
         return None
@@ -1239,8 +1245,8 @@ class Logging(LiteLLMLoggingBaseClass):
                 return {"error": f"Unable to parse raw request body. Got - {data}"}
         return data
 
-    def _get_masked_api_base(self, api_base: str) -> str:
-        return str(mask_api_base_credentials(api_base))
+    def _get_masked_api_base(self, api_base: str | None) -> str:
+        return str(mask_api_base_credentials(api_base or ""))
 
     def _pre_call(self, input, api_key, model=None, additional_args={}):
         """

litellm/litellm_core_utils/llm_response_utils/get_formatted_prompt.py

@@ -30,7 +30,7 @@ def get_formatted_prompt(
                         if c["type"] == "text":
                             prompt += c["text"]
             if "tool_calls" in message:
-                for tool_call in message["tool_calls"]:
+                for tool_call in message["tool_calls"] or ():
                     if "function" in tool_call:
                         function_arguments = tool_call["function"]["arguments"]
                         prompt += function_arguments

litellm/litellm_core_utils/sensitive_data_masker.py

@@ -1,5 +1,6 @@
 from collections.abc import Mapping, Sequence
 from collections.abc import Set as AbstractSet
+from dataclasses import dataclass, field
 from typing import Any, Final
 
 from pydantic import BaseModel
@@ -176,26 +177,49 @@ def mask_credentials_in_payload(data: object) -> object:
     config-dump semantics (``None`` -> ``"None"``, tuples stringified,
     objects flattened via ``__dict__``) would silently distort the record.
 
+    A container referenced from several places in ``data`` is rebuilt once and
+    referenced from the same places in the copy, so a shared subtree never
+    fans out into independent copies, and a reference back into a container
+    still being rebuilt (a cycle) becomes ``REDACTED``. A container nested past
+    ``DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER`` is replaced by
+    ``REDACTED`` rather than returned unmasked.
+
     Sensitive-key detection is delegated to the shared
     :class:`SensitiveDataMasker` so pattern updates stay in one place.
     """
-    return _walk_payload(data, key_is_sensitive=False, depth=0)
+    return _PayloadWalker().walk(data, key_is_sensitive=False, depth=0)
 
 
-def _walk_payload(node: object, key_is_sensitive: bool, depth: int) -> object:
-    if depth >= DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER:
-        return node
-    if isinstance(node, Mapping):
-        return {k: _walk_payload(v, _default_masker.is_sensitive_key(k), depth + 1) for k, v in node.items()}
-    if isinstance(node, list):
-        return [_walk_payload(item, key_is_sensitive, depth + 1) for item in node]
-    if isinstance(node, tuple):
-        return tuple(_walk_payload(item, key_is_sensitive, depth + 1) for item in node)
-    if isinstance(node, BaseModel):
-        return _walk_payload(node.model_dump(), key_is_sensitive, depth)
-    if key_is_sensitive and isinstance(node, str) and node:
-        return _default_masker._mask_value(node)
-    return node
+@dataclass(frozen=True, slots=True)
+class _PayloadWalker:
+    _memo: dict[tuple[int, bool], tuple[object, object]] = field(  # mutable-ok: memo of one walk, pins each keyed node
+        default_factory=dict
+    )
+
+    def walk(self, node: object, key_is_sensitive: bool, depth: int) -> object:
+        if not isinstance(node, (Mapping, list, tuple, BaseModel)):
+            return _default_masker._mask_value(node) if key_is_sensitive and isinstance(node, str) and node else node
+        if depth >= DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER:
+            return REDACTED
+        memo_key: Final = (id(node), key_is_sensitive and not isinstance(node, Mapping))
+        cached: Final = self._memo.get(memo_key)
+        if cached is not None:
+            return cached[1]
+        self._memo[memo_key] = (node, REDACTED)
+        rebuilt: Final = self._rebuild(node, key_is_sensitive, depth)
+        self._memo[memo_key] = (node, rebuilt)
+        return rebuilt
+
+    def _rebuild(
+        self, node: Mapping[str, object] | Sequence[object] | BaseModel, key_is_sensitive: bool, depth: int
+    ) -> object:
+        if isinstance(node, BaseModel):
+            return self.walk(node.model_dump(), key_is_sensitive, depth)
+        if isinstance(node, Mapping):
+            return {k: self.walk(v, _default_masker.is_sensitive_key(k), depth + 1) for k, v in node.items()}
+        if isinstance(node, tuple):
+            return tuple(self.walk(item, key_is_sensitive, depth + 1) for item in node)
+        return [self.walk(item, key_is_sensitive, depth + 1) for item in node]
 
 
 def mask_sensitive_keys(data: dict[str, Any], sensitive_fields: set[str]) -> dict[str, Any]:

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -1253,10 +1253,9 @@ class AnthropicMessagesHandler(BaseTranslation):
         Process output streaming response by applying guardrails to text content.
 
         Get the string so far, check the apply guardrail to the string so far, and return the list of responses so far.
-        With ``deliver_ended_stream_rewrites``, an ended stream whose guardrail rewrote the text gets the rewrite
-        written back across the buffered chunks (full rewritten text in the first ``text_delta``, the rest blanked);
-        a rewrite on a stream that never reported a ``stop_reason`` has no write-back and is reported as
-        undeliverable, so the pipeline executor discards it and releases the original chunks.
+        With ``deliver_ended_stream_rewrites``, a stream whose guardrail rewrote the text gets the rewrite
+        written back across the buffered chunks (full rewritten text in the first ``text_delta``, the rest blanked),
+        whether or not the stream ever reported a ``stop_reason``.
         """
         from litellm.integrations.custom_guardrail import ModifyResponseException
 
@@ -1312,7 +1311,11 @@ class AnthropicMessagesHandler(BaseTranslation):
                     and guardrailed_texts
                     and guardrailed_texts[0] != string_so_far
                 ):
-                    self._write_ended_stream_text_rewrite(responses_so_far, guardrailed_texts[0])
+                    self._write_ended_stream_text_rewrite(
+                        responses_so_far,
+                        guardrailed_texts[0],
+                        guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
+                    )
                 if deliver_ended_stream_rewrites:
                     returned_tool_calls: Final = _guardrailed_inputs.get("tool_calls")
                     self._write_ended_stream_tool_call_rewrites(
@@ -1354,9 +1357,11 @@ class AnthropicMessagesHandler(BaseTranslation):
             raise
         unended_texts: Final = _guardrailed_inputs.get("texts")
         if deliver_ended_stream_rewrites and unended_texts and tuple(unended_texts) != (string_so_far,):
-            from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
-
-            raise UndeliverableStreamRewrite(guardrail_to_apply.guardrail_name or "unknown")
+            self._write_ended_stream_text_rewrite(
+                responses_so_far,
+                unended_texts[0],
+                guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
+            )
         return responses_so_far
 
     def _prepare_request_data(
@@ -1450,26 +1455,40 @@ class AnthropicMessagesHandler(BaseTranslation):
             inputs["model"] = response_model
         return inputs
 
-    @staticmethod
+    @classmethod
     def _write_ended_stream_text_rewrite(
+        cls,
         responses_so_far: MutableSequence[object],  # mutable-ok: rewrites the caller's buffered chunks in place
         rewritten_text: str,
+        guardrail_name: str,
     ) -> None:
         """Deliver an ended-stream guardrail text rewrite by rewriting the
         buffered chunks in place: the first ``text_delta`` carries the full
         rewritten text and every later one is blanked, leaving the surrounding
-        message and content-block framing untouched."""
+        message and content-block framing untouched. A buffer with no
+        ``text_delta`` has nowhere to carry the rewrite, so the pipeline
+        executor discards it and releases the original chunks."""
+
+        def is_text_delta(event: Mapping[str, object]) -> bool:
+            delta: Final = event.get("delta")
+            return (
+                event.get("type") == "content_block_delta"
+                and isinstance(delta, Mapping)
+                and delta.get("type") == "text_delta"
+            )
+
+        if not any(is_text_delta(event) for item in responses_so_far for event in cls._iter_sse_events(item)):
+            from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
+
+            raise UndeliverableStreamRewrite(guardrail_name)
         replacements: Final = chain((rewritten_text,), repeat(""))
 
         def rewrite_text_delta(event: Mapping[str, object]) -> _SSEFieldRewrite | None:
-            delta: Final = event.get("delta")
-            if event.get("type") != "content_block_delta" or not isinstance(delta, Mapping):
-                return None
-            if delta.get("type") != "text_delta":
+            if not is_text_delta(event):
                 return None
             return _SSEFieldRewrite("delta", "text", next(replacements))
 
-        AnthropicMessagesHandler._rewrite_ended_stream_events(responses_so_far, rewrite_text_delta)
+        cls._rewrite_ended_stream_events(responses_so_far, rewrite_text_delta)
 
     @classmethod
     def _write_ended_stream_tool_call_rewrites(

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -384,7 +384,7 @@ class LiteLLMAnthropicMessagesAdapter:
         cache_control: Final = (
             source.get("cache_control") if isinstance(source, dict) else getattr(source, "cache_control", None)
         )
-        if cache_control and model and (self.is_anthropic_claude_model(model) or self.is_bedrock_arn_model(model)):
+        if cache_control and model and self.target_consumes_cache_control(model):
             # TypedDict objects support dict operations at runtime
             # Use type ignore consistent with codebase pattern (see anthropic/chat/transformation.py:432)
             if isinstance(target, dict):
@@ -677,6 +677,10 @@ class LiteLLMAnthropicMessagesAdapter:
         model_lower: Final = model.lower()
         return "arn:" in model_lower and ":bedrock:" in model_lower
 
+    @classmethod
+    def target_consumes_cache_control(cls, model: str) -> bool:
+        return cls.is_anthropic_claude_model(model) or cls.is_bedrock_arn_model(model) or "gemini" in model.lower()
+
     @staticmethod
     def translate_thinking_for_model(
         thinking: AnthropicThinkingParam,

litellm/llms/azure/azure.py

@@ -46,7 +46,9 @@ from .common_utils import (
     AzureOpenAIError,
     BaseAzureLLM,
     get_azure_ad_token_from_oidc,
+    get_azure_request_auth_headers,
     process_azure_headers,
+    redact_azure_auth_headers,
     select_azure_base_url_or_endpoint,
 )
 from .image_generation import (
@@ -1145,7 +1147,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         api_key: str,
         input: list,
         logging_obj: LiteLLMLoggingObj,
-        headers: dict,
+        headers: dict[str, str],
         client=None,
         timeout=None,
         model: str | None = None,
@@ -1170,7 +1172,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 additional_args={
                     "complete_input_dict": data,
                     "api_base": img_gen_api_base,
-                    "headers": headers,
+                    "headers": redact_azure_auth_headers(headers),
                 },
             )
             httpx_response: Final[httpx.Response] = await self.make_async_azure_httpx_request(
@@ -1229,7 +1231,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         timeout: float,
         optional_params: dict,
         logging_obj: LiteLLMLoggingObj,
-        headers: dict,
+        headers: dict[str, str],
         model: str | None = None,
         api_key: str | None = None,
         api_base: str | None = None,
@@ -1264,21 +1266,22 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             if not isinstance(max_retries, int):
                 raise AzureOpenAIError(status_code=422, message="max retries must be an int")
 
-            if api_key is None and azure_ad_token_provider is not None:
-                azure_ad_token = azure_ad_token_provider()
-                if azure_ad_token:
-                    headers.pop("api-key", None)
-                    headers["Authorization"] = f"Bearer {azure_ad_token}"
-
-            # init AzureOpenAI Client
+            auth_params: Final[dict[str, object]] = {**(litellm_params or {})}  # mutable-ok: SDK init takes a dict
+            if azure_ad_token is not None:
+                auth_params["azure_ad_token"] = azure_ad_token
+            if azure_ad_token_provider is not None:
+                auth_params["azure_ad_token_provider"] = azure_ad_token_provider
             azure_client_params: Final[dict[str, object]] = self.initialize_azure_sdk_client(
-                litellm_params=litellm_params or {},
+                litellm_params=auth_params,
                 api_key=api_key,
                 model_name=model or "",
                 api_version=api_version,
                 api_base=api_base,
                 is_async=False,
             )
+            request_headers: Final = dict(  # mutable-ok: the httpx request helpers take a dict
+                get_azure_request_auth_headers(headers=headers, azure_client_params=azure_client_params)
+            )
             if aimg_generation is True:
                 return self.aimage_generation(
                     data=data,
@@ -1289,7 +1292,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                     client=client,
                     azure_client_params=azure_client_params,
                     timeout=timeout,
-                    headers=headers,
+                    headers=request_headers,
                     model=model,
                 )
 
@@ -1306,7 +1309,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 additional_args={
                     "complete_input_dict": data,
                     "api_base": img_gen_api_base,
-                    "headers": headers,
+                    "headers": redact_azure_auth_headers(request_headers),
                 },
             )
             httpx_response: Final[httpx.Response] = self.make_sync_azure_httpx_request(
@@ -1316,7 +1319,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 api_version=api_version or "",
                 api_key=api_key or "",
                 data=data,
-                headers=headers,
+                headers=request_headers,
                 deployment_name=model,
             )
             provider_config: Final = get_azure_image_generation_config(data.get("model", "dall-e-2"))

litellm/llms/azure/common_utils.py

@@ -96,6 +96,11 @@ def _cached_entra_id_token_provider(
     return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope)
 
 
+@lru_cache(maxsize=128)
+def _cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]:
+    return get_azure_ad_token_provider(azure_scope=scope)
+
+
 def get_azure_ad_token_from_entra_id(
     tenant_id: str,
     client_id: str,
@@ -406,6 +411,41 @@ def get_azure_ad_token(
     return azure_ad_token
 
 
+_AZURE_AUTH_HEADER_NAMES: Final = frozenset(("api-key", "authorization"))
+_REDACTED_AZURE_HEADER_VALUE: Final = "***REDACTED***"
+
+
+def _resolve_azure_ad_token(azure_client_params: Mapping[str, object]) -> str | None:
+    azure_ad_token: Final = azure_client_params.get("azure_ad_token")
+    if isinstance(azure_ad_token, str) and azure_ad_token:
+        return azure_ad_token
+    token_provider: Final = azure_client_params.get("azure_ad_token_provider")
+    provided_token: Final = token_provider() if callable(token_provider) else None
+    return provided_token if isinstance(provided_token, str) and provided_token else None
+
+
+def get_azure_request_auth_headers(
+    headers: Mapping[str, str],
+    azure_client_params: Mapping[str, object],
+) -> Mapping[str, str]:
+    if any(name.lower() in _AZURE_AUTH_HEADER_NAMES for name in headers):
+        return headers
+    azure_ad_token: Final = _resolve_azure_ad_token(azure_client_params)
+    if azure_ad_token is not None:
+        return MappingProxyType({**headers, "Authorization": f"Bearer {azure_ad_token}"})
+    api_key: Final = azure_client_params.get("api_key")
+    if isinstance(api_key, str) and api_key:
+        return MappingProxyType({**headers, "api-key": api_key})
+    return headers
+
+
+def redact_azure_auth_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
+    return {  # mutable-ok: logging callbacks JSON-serialize this copy
+        name: (_REDACTED_AZURE_HEADER_VALUE if name.lower() in _AZURE_AUTH_HEADER_NAMES else value)
+        for name, value in headers.items()
+    }
+
+
 class BaseAzureLLM(BaseOpenAILLM):
     @staticmethod
     def _try_get_default_azure_credential_provider(
@@ -616,9 +656,7 @@ class BaseAzureLLM(BaseOpenAILLM):
                 "Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth"
             )
             try:
-                azure_ad_token_provider = get_azure_ad_token_provider(
-                    azure_scope=scope,
-                )
+                azure_ad_token_provider = _cached_azure_ad_token_refresh_provider(scope)
             except ValueError:
                 verbose_logger.debug("Azure AD Token Provider could not be used.")
         if api_version is None:

litellm/llms/base_llm/search/transformation.py

@@ -95,6 +95,18 @@ class BaseSearchConfig:
         """
         return "Unknown Search Provider"
 
+    def supports_rich_search_input(self) -> bool:
+        """
+        Whether this provider's search API accepts a natural-language
+        objective plus multiple keyword queries in one request.
+
+        Integrations that collect the richer shape (e.g. websearch
+        interception) forward ``query`` as a list plus an ``objective``
+        optional param to providers that return True; every other provider
+        keeps receiving the single query string.
+        """
+        return False
+
     def get_http_method(self) -> Literal["GET", "POST"]:
         """
         Get HTTP method for search requests.

litellm/llms/bedrock/chat/agentcore/transformation.py

@@ -6,7 +6,7 @@ https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgen
 
 import json
 from collections.abc import AsyncGenerator
-from typing import TYPE_CHECKING, Any, Final, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Optional, Union
 from urllib.parse import quote
 
 import httpx
@@ -31,6 +31,7 @@ from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import (
     Choices,
     Delta,
+    LlmProviders,
     Message,
     ModelResponse,
     ModelResponseStream,
@@ -872,7 +873,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         )
 
         if client is None or not isinstance(client, AsyncHTTPHandler):
-            client = get_async_httpx_client(llm_provider=cast(Any, "bedrock"), params={})
+            client = get_async_httpx_client(llm_provider=LlmProviders.BEDROCK, params={})
 
         verbose_logger.debug("Making async streaming request to: %s", api_base)
 

litellm/llms/custom_httpx/llm_http_handler.py

@@ -293,6 +293,26 @@ def _has_pre_call_deployment_hook(logging_obj: LiteLLMLoggingObj) -> bool:
     return False
 
 
+def _mask_presigned_request_headers(transformed_request: bytes | str | dict) -> bytes | str | dict:
+    """A pre-signed request carries its auth inside its own ``headers`` key, which
+    logging treats as request body (only the top-level headers channel gets masked),
+    so mask it here before the request is handed to ``pre_call``."""
+    if not isinstance(transformed_request, dict):
+        return transformed_request
+    request_headers: Final = transformed_request.get("headers")
+    if not isinstance(request_headers, dict):
+        return transformed_request
+
+    from litellm.litellm_core_utils.litellm_logging import (
+        _get_masked_values,  # pyright: ignore[reportPrivateUsage]  # the shared header-masking helper has no public name
+    )
+
+    return {  # mutable-ok: logging's curl and raw-request builders take dict
+        **transformed_request,
+        "headers": _get_masked_values(request_headers),
+    }
+
+
 def _aws_signing_overrides(optional_params: Mapping[str, Any], litellm_params: Mapping[str, Any]) -> Mapping[str, Any]:
     return MappingProxyType(
         {
@@ -3734,7 +3754,7 @@ class BaseLLMHTTPHandler:
                 "complete_input_dict": (
                     "<streaming media upload>"
                     if isinstance(transformed_request, dict) and "streaming_media_upload" in transformed_request
-                    else transformed_request
+                    else _mask_presigned_request_headers(transformed_request)
                 ),
                 "api_base": api_base,
                 "headers": headers,
@@ -4157,7 +4177,7 @@ class BaseLLMHTTPHandler:
             input="",
             api_key="",
             additional_args={
-                "complete_input_dict": transformed_request,
+                "complete_input_dict": _mask_presigned_request_headers(transformed_request),
                 "api_base": api_base,
                 "headers": headers,
             },
@@ -4236,7 +4256,7 @@ class BaseLLMHTTPHandler:
             input="",
             api_key="",
             additional_args={
-                "complete_input_dict": transformed_request,
+                "complete_input_dict": _mask_presigned_request_headers(transformed_request),
                 "api_base": api_base,
                 "headers": headers,
                 "batch_id": batch_id,

litellm/llms/mistral/batches/transformation.py

@@ -0,0 +1,220 @@
+"""
+Mistral Batch API. Reference: https://docs.mistral.ai/api/#tag/batch
+
+Mistral runs one model per job (set on the job, not per input line) and accepts
+``/v1/ocr`` as a batch endpoint, which is how OCR gets its 50% batch discount.
+Output and error files are OpenAI-shaped JSONL (``{custom_id, response: {status_code, body}}``),
+so the shared batch cost accounting reads them without a provider branch.
+"""
+
+from collections.abc import Mapping, Sequence
+from types import MappingProxyType
+from typing import Final, Literal, TypeAlias
+
+import httpx
+from openai.types.batch import BatchRequestCounts
+from openai.types.batch import Errors as BatchErrors
+from openai.types.batch_error import BatchError
+from pydantic import BaseModel, ConfigDict
+from typing_extensions import NotRequired, ReadOnly, TypedDict
+
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
+from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest
+from litellm.types.utils import LiteLLMBatch, LlmProviders
+
+from ..common_utils import get_mistral_api_base, get_mistral_auth_headers, mistral_error
+
+MistralBatchStatus: TypeAlias = Literal[
+    "QUEUED", "RUNNING", "SUCCESS", "FAILED", "TIMEOUT_EXCEEDED", "CANCELLATION_REQUESTED", "CANCELLED"
+]
+OpenAIBatchStatus: TypeAlias = Literal[
+    "validating", "failed", "in_progress", "finalizing", "completed", "expired", "cancelling", "cancelled"
+]
+
+_NO_HEADERS: Final[Mapping[str, str]] = MappingProxyType({})  # mutable-ok: frozen at module scope
+_STATUS_MAP: Final[MappingProxyType[MistralBatchStatus, OpenAIBatchStatus]] = MappingProxyType(
+    {
+        "QUEUED": "validating",
+        "RUNNING": "in_progress",
+        "SUCCESS": "completed",
+        "FAILED": "failed",
+        "TIMEOUT_EXCEEDED": "expired",
+        "CANCELLATION_REQUESTED": "cancelling",
+        "CANCELLED": "cancelled",
+    }
+)
+
+
+class MistralCreateBatchJobRequest(TypedDict):
+    """Body of ``POST /v1/batch/jobs``."""
+
+    input_files: ReadOnly[tuple[str, ...]]
+    endpoint: ReadOnly[str]
+    model: ReadOnly[str]
+    metadata: NotRequired[ReadOnly[Mapping[str, str]]]
+
+
+class MistralPresignedRequest(TypedDict):
+    """A fully-formed request the shared HTTP handler sends as-is (its ``method`` branch)."""
+
+    method: ReadOnly[Literal["GET"]]
+    url: ReadOnly[str]
+    headers: ReadOnly[Mapping[str, str]]
+
+
+class MistralBatchError(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    message: str
+    count: int = 1
+
+
+class MistralBatchJob(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    id: str
+    input_files: tuple[str, ...] = ()
+    endpoint: str
+    model: str | None = None
+    status: MistralBatchStatus
+    created_at: int
+    started_at: int | None = None
+    completed_at: int | None = None
+    total_requests: int = 0
+    completed_requests: int = 0
+    succeeded_requests: int = 0
+    failed_requests: int = 0
+    output_file: str | None = None
+    error_file: str | None = None
+    errors: tuple[MistralBatchError, ...] = ()
+    metadata: dict[str, str] | None = None  # mutable-ok: LiteLLMBatch.metadata is typed as dict
+
+
+def _to_batch_errors(errors: Sequence[MistralBatchError]) -> BatchErrors | None:
+    if not errors:
+        return None
+    return BatchErrors(
+        object="list",
+        data=[  # mutable-ok: openai Batch.Errors.data is typed as list
+            BatchError(message=f"{e.message} (x{e.count})" if e.count > 1 else e.message) for e in errors
+        ],
+    )
+
+
+def _to_litellm_batch(job: MistralBatchJob) -> LiteLLMBatch:
+    status: Final = _STATUS_MAP[job.status]
+    terminal_at: Final = job.completed_at
+    return LiteLLMBatch(
+        id=job.id,
+        object="batch",
+        endpoint=job.endpoint,
+        input_file_id=job.input_files[0] if job.input_files else "",
+        completion_window="24h",
+        status=status,
+        created_at=job.created_at,
+        in_progress_at=job.started_at,
+        completed_at=terminal_at if status == "completed" else None,
+        failed_at=terminal_at if status == "failed" else None,
+        expired_at=terminal_at if status == "expired" else None,
+        cancelled_at=terminal_at if status == "cancelled" else None,
+        output_file_id=job.output_file,
+        error_file_id=job.error_file,
+        errors=_to_batch_errors(job.errors),
+        request_counts=BatchRequestCounts(
+            total=job.total_requests,
+            completed=job.succeeded_requests,
+            failed=job.failed_requests,
+        ),
+        metadata=job.metadata,
+    )
+
+
+class MistralBatchesConfig(BaseBatchesConfig):
+    @property
+    def custom_llm_provider(self) -> LlmProviders:
+        return LlmProviders.MISTRAL
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[AllMessageValues],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: BaseBatchesConfig signature
+        return get_mistral_auth_headers(headers, api_key)
+
+    def get_complete_batch_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        data: CreateBatchRequest,
+    ) -> str:
+        return f"{get_mistral_api_base(api_base)}/v1/batch/jobs"
+
+    def transform_create_batch_request(
+        self,
+        model: str,
+        create_batch_data: CreateBatchRequest,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> dict[str, object]:  # mutable-ok: BaseBatchesConfig signature
+        input_file_id: Final = create_batch_data.get("input_file_id")
+        endpoint: Final = create_batch_data.get("endpoint")
+        if input_file_id is None or endpoint is None:
+            raise ValueError("input_file_id and endpoint are required to create a Mistral batch job")
+        metadata: Final = create_batch_data.get("metadata")
+        body: Final = (
+            MistralCreateBatchJobRequest(
+                input_files=(input_file_id,), endpoint=endpoint, model=model, metadata=metadata
+            )
+            if metadata
+            else MistralCreateBatchJobRequest(input_files=(input_file_id,), endpoint=endpoint, model=model)
+        )
+        return dict(body)  # mutable-ok: BaseBatchesConfig signature
+
+    def transform_create_batch_response(
+        self,
+        model: str | None,
+        raw_response: httpx.Response,
+        logging_obj: object,
+        litellm_params: Mapping[str, object],
+    ) -> LiteLLMBatch:
+        return _to_litellm_batch(MistralBatchJob.model_validate(raw_response.json()))
+
+    def transform_retrieve_batch_request(
+        self,
+        batch_id: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> dict[str, object]:  # mutable-ok: BaseBatchesConfig signature
+        encoded_batch_id: Final = encode_url_path_segment(batch_id, field_name="batch_id")
+        api_base: Final = litellm_params.get("api_base")
+        api_key: Final = litellm_params.get("api_key")
+        request: Final = MistralPresignedRequest(
+            method="GET",
+            url=f"{get_mistral_api_base(api_base if isinstance(api_base, str) else None)}/v1/batch/jobs/{encoded_batch_id}",
+            headers=get_mistral_auth_headers(_NO_HEADERS, api_key if isinstance(api_key, str) else None),
+        )
+        return dict(request)  # mutable-ok: BaseBatchesConfig signature
+
+    def transform_retrieve_batch_response(
+        self,
+        model: str | None,
+        raw_response: httpx.Response,
+        logging_obj: object,
+        litellm_params: Mapping[str, object],
+    ) -> LiteLLMBatch:
+        return _to_litellm_batch(MistralBatchJob.model_validate(raw_response.json()))
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers
+    ) -> BaseLLMException:
+        return mistral_error(error_message, status_code, headers)

litellm/llms/mistral/common_utils.py

@@ -0,0 +1,41 @@
+from collections.abc import Mapping
+from typing import Final
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.secret_managers.main import get_secret_str
+
+MISTRAL_API_BASE: Final = "https://api.mistral.ai"
+MISTRAL_API_KEY_ENV_VAR: Final = "MISTRAL_API_KEY"
+
+
+class MistralError(BaseLLMException):
+    pass
+
+
+def get_mistral_api_base(api_base: str | None) -> str:
+    """Return the Mistral origin without a trailing ``/v1``, so callers can append ``/v1/<route>``."""
+    resolved: Final = (api_base or get_secret_str("MISTRAL_API_BASE") or MISTRAL_API_BASE).rstrip("/")
+    return resolved.removesuffix("/v1")
+
+
+def get_mistral_auth_headers(
+    headers: Mapping[str, str], api_key: str | None
+) -> dict[str, str]:  # mutable-ok: BaseConfig.validate_environment contract returns dict
+    resolved_key: Final = api_key or get_secret_str(MISTRAL_API_KEY_ENV_VAR)
+    if resolved_key is None:
+        raise ValueError(
+            "Missing Mistral API Key - A call is being made to Mistral but no key is set either in the environment variables or via params"
+        )
+    return dict(headers, Authorization=f"Bearer {resolved_key}")  # mutable-ok: BaseConfig contract returns dict
+
+
+def mistral_error(error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers) -> MistralError:
+    return MistralError(
+        status_code=status_code,
+        message=error_message,
+        headers=headers
+        if isinstance(headers, httpx.Headers)
+        else httpx.Headers(dict(headers)),  # mutable-ok: httpx.Headers takes a dict
+    )

litellm/llms/mistral/files/transformation.py

@@ -0,0 +1,267 @@
+"""
+Mistral Files API. Reference: https://docs.mistral.ai/api/#tag/files
+
+Mistral's file objects already carry the OpenAI field names (id, bytes, created_at,
+filename, purpose), so this config is URL routing, auth, and a purpose mapping:
+Mistral only accepts ``fine-tune``, ``batch`` and ``ocr`` as upload purposes, while files
+other Mistral products created read back with purposes outside that set and map onto ``user_data``.
+"""
+
+import time
+from collections.abc import Mapping, Sequence
+from types import MappingProxyType
+from typing import Final, Literal, TypeAlias
+
+import httpx
+from openai.types.file_deleted import FileDeleted
+from pydantic import BaseModel, ConfigDict
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.files.transformation import BaseFilesConfig, LiteLLMLoggingObj
+from litellm.types.llms.openai import (
+    CreateFileRequest,
+    FileContentRequest,
+    HttpxBinaryResponseContent,
+    OpenAICreateFileRequestOptionalParams,
+    OpenAIFileObject,
+    OpenAIFilesPurpose,
+)
+from litellm.types.utils import LlmProviders
+
+from ..common_utils import get_mistral_api_base, get_mistral_auth_headers, mistral_error
+
+MistralFilePurpose: TypeAlias = Literal["fine-tune", "batch", "ocr"]
+
+_OPENAI_PURPOSE_BY_MISTRAL: Final[Mapping[str, OpenAIFilesPurpose]] = MappingProxyType(
+    {"fine-tune": "fine-tune", "batch": "batch", "ocr": "user_data"}
+)
+_OPENAI_PURPOSE_FOR_UNMAPPED: Final[OpenAIFilesPurpose] = "user_data"
+_MISTRAL_PURPOSE_BY_OPENAI: Final[Mapping[str, MistralFilePurpose]] = MappingProxyType(
+    {"fine-tune": "fine-tune", "batch": "batch", "ocr": "ocr", "user_data": "ocr"}
+)
+_SUPPORTED_PURPOSES: Final = ", ".join(_MISTRAL_PURPOSE_BY_OPENAI)
+
+_NO_QUERY_PARAMS: Final[dict[str, str]] = {}  # mutable-ok: BaseFilesConfig request transforms return tuple[str, dict]
+
+
+class MistralMultipartUpload(TypedDict):
+    """``files=`` payload for ``POST /v1/files``: each value is an httpx multipart tuple."""
+
+    file: ReadOnly[tuple[str, object, str]]
+    purpose: ReadOnly[tuple[None, MistralFilePurpose]]
+
+
+class MistralFile(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    id: str
+    bytes: int = 0
+    created_at: int | None = None
+    filename: str = ""
+    purpose: str = "batch"
+    expires_at: int | None = None
+
+
+class MistralFileList(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    data: tuple[MistralFile, ...] = ()
+
+
+class MistralFileDeleted(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore")
+
+    id: str
+    deleted: bool = True
+
+
+def _to_openai_file_object(file: MistralFile) -> OpenAIFileObject:
+    return OpenAIFileObject(
+        id=file.id,
+        bytes=file.bytes,
+        created_at=file.created_at if file.created_at is not None else int(time.time()),
+        filename=file.filename,
+        object="file",
+        purpose=_to_openai_purpose(file.purpose),
+        status="uploaded",
+        expires_at=file.expires_at,
+    )
+
+
+def _to_openai_purpose(purpose: str) -> OpenAIFilesPurpose:
+    return _OPENAI_PURPOSE_BY_MISTRAL.get(purpose, _OPENAI_PURPOSE_FOR_UNMAPPED)
+
+
+def _to_mistral_purpose(purpose: str) -> MistralFilePurpose:
+    """``user_data`` is what an OCR file reads back as, since OpenAI's purpose literal has no ``ocr``,
+    so it maps back onto ``ocr``. Every other purpose Mistral lacks is rejected: silently rewriting
+    it to ``batch`` would let an upload skip the proxy's batch-file validation and guardrails, which
+    only run when the caller says ``purpose=batch``."""
+    mistral_purpose: Final = _MISTRAL_PURPOSE_BY_OPENAI.get(purpose)
+    if mistral_purpose is None:
+        raise mistral_error(
+            f"Mistral does not support purpose={purpose!r}. Use one of: {_SUPPORTED_PURPOSES}",
+            status_code=400,
+            headers=httpx.Headers(),
+        )
+    return mistral_purpose
+
+
+def _api_base_from(litellm_params: Mapping[str, object]) -> str:
+    api_base: Final = litellm_params.get("api_base")
+    return get_mistral_api_base(api_base if isinstance(api_base, str) else None)
+
+
+class MistralFilesConfig(BaseFilesConfig):
+    @property
+    def custom_llm_provider(self) -> LlmProviders:
+        return LlmProviders.MISTRAL
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        api_key: str | None,
+        model: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        stream: bool | None = None,
+    ) -> str:
+        return f"{get_mistral_api_base(api_base)}/v1/files"
+
+    def _file_url(self, file_id: str, litellm_params: Mapping[str, object], suffix: str = "") -> str:
+        encoded_file_id: Final = encode_url_path_segment(file_id, field_name="file_id")
+        return f"{_api_base_from(litellm_params)}/v1/files/{encoded_file_id}{suffix}"
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Mapping[str, str] | httpx.Headers
+    ) -> BaseLLMException:
+        return mistral_error(error_message, status_code, headers)
+
+    def validate_environment(
+        self,
+        headers: Mapping[str, str],
+        model: str,
+        messages: Sequence[object],
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+        api_key: str | None = None,
+        api_base: str | None = None,
+    ) -> dict[str, str]:  # mutable-ok: BaseFilesConfig signature
+        return get_mistral_auth_headers(headers, api_key)
+
+    def get_supported_openai_params(
+        self, model: str
+    ) -> list[OpenAICreateFileRequestOptionalParams]:  # mutable-ok: BaseFilesConfig signature
+        return ["purpose"]  # mutable-ok: BaseFilesConfig signature
+
+    def map_openai_params(
+        self,
+        non_default_params: Mapping[str, object],
+        optional_params: dict[str, object],  # mutable-ok: BaseConfig signature, returned as-is
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: BaseConfig signature
+        return optional_params
+
+    def transform_create_file_request(
+        self,
+        model: str,
+        create_file_data: CreateFileRequest,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> dict[str, object]:  # mutable-ok: BaseFilesConfig signature
+        if "file" not in create_file_data:
+            raise ValueError("File data is required")
+        extracted: Final = extract_file_data(create_file_data["file"])
+        filename: Final = extracted["filename"] or f"file_{int(time.time())}.jsonl"
+        content_type: Final = extracted.get("content_type") or "application/octet-stream"
+        upload: Final = MistralMultipartUpload(
+            file=(filename, extracted["content"], content_type),
+            purpose=(None, _to_mistral_purpose(create_file_data.get("purpose") or "batch")),
+        )
+        return dict(upload)  # mutable-ok: BaseFilesConfig signature
+
+    def transform_create_file_response(
+        self,
+        model: str | None,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        litellm_params: Mapping[str, object],
+    ) -> OpenAIFileObject:
+        return _to_openai_file_object(MistralFile.model_validate(raw_response.json()))
+
+    def transform_retrieve_file_request(
+        self,
+        file_id: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> tuple[str, dict[str, str]]:  # mutable-ok: BaseFilesConfig signature
+        return self._file_url(file_id, litellm_params), _NO_QUERY_PARAMS
+
+    def transform_retrieve_file_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        litellm_params: Mapping[str, object],
+    ) -> OpenAIFileObject:
+        return _to_openai_file_object(MistralFile.model_validate(raw_response.json()))
+
+    def transform_delete_file_request(
+        self,
+        file_id: str,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> tuple[str, dict[str, str]]:  # mutable-ok: BaseFilesConfig signature
+        return self._file_url(file_id, litellm_params), _NO_QUERY_PARAMS
+
+    def transform_delete_file_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        litellm_params: Mapping[str, object],
+    ) -> FileDeleted:
+        deleted: Final = MistralFileDeleted.model_validate(raw_response.json())
+        return FileDeleted(id=deleted.id, deleted=deleted.deleted, object="file")
+
+    def transform_list_files_request(
+        self,
+        purpose: str | None,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> tuple[str, dict[str, str]]:  # mutable-ok: BaseFilesConfig signature
+        url: Final = f"{_api_base_from(litellm_params)}/v1/files"
+        if not purpose:
+            return url, _NO_QUERY_PARAMS
+        return url, {"purpose": _to_mistral_purpose(purpose)}  # mutable-ok: BaseFilesConfig signature returns dict
+
+    def transform_list_files_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        litellm_params: Mapping[str, object],
+    ) -> list[OpenAIFileObject]:  # mutable-ok: BaseFilesConfig signature
+        return [  # mutable-ok: BaseFilesConfig signature
+            _to_openai_file_object(f) for f in MistralFileList.model_validate(raw_response.json()).data
+        ]
+
+    def transform_file_content_request(
+        self,
+        file_content_request: FileContentRequest,
+        optional_params: Mapping[str, object],
+        litellm_params: Mapping[str, object],
+    ) -> tuple[str, dict[str, str]]:  # mutable-ok: BaseFilesConfig signature
+        file_id: Final = file_content_request.get("file_id")
+        if file_id is None:
+            raise ValueError("file_id is required to download file content")
+        return self._file_url(file_id, litellm_params, suffix="/content"), _NO_QUERY_PARAMS
+
+    def transform_file_content_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        litellm_params: Mapping[str, object],
+    ) -> HttpxBinaryResponseContent:
+        return HttpxBinaryResponseContent(response=raw_response)

litellm/llms/openai/chat/guardrail_translation/handler.py

@@ -18,6 +18,7 @@ import json
 import time
 import uuid
 from collections.abc import Sequence
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Union, cast
 
 from typing_extensions import NotRequired, ReadOnly, TypedDict
@@ -651,10 +652,7 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         """Ended-stream path: rebuild the full response, run the non-streaming
         output guardrail against it, and (when opted in) write any text or
         tool-call rewrite back across the buffered chunks."""
-        model_response: Final = cast(
-            ModelResponse,
-            stream_chunk_builder(chunks=responses_so_far, logging_obj=litellm_logging_obj),
-        )
+        model_response: Final = self._rebuild_ended_stream_per_choice(responses_so_far, litellm_logging_obj)
         pre_guardrail_texts: Final = self._string_choice_contents(model_response)
         pre_guardrail_tool_calls: Final = self._function_tool_call_shapes(model_response)
         await self.process_output_response(
@@ -666,20 +664,59 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         )
         if not deliver_ended_stream_rewrites:
             return
-        guardrail_name: Final = guardrail_to_apply.guardrail_name or "unknown"
         await self._write_ended_stream_text_rewrites(
             responses_so_far=responses_so_far,
             guardrailed_response=model_response,
             pre_guardrail_texts=pre_guardrail_texts,
-            guardrail_name=guardrail_name,
         )
         self._write_ended_stream_tool_call_rewrites(
             responses_so_far=responses_so_far,
             guardrailed_response=model_response,
             pre_guardrail_tool_calls=pre_guardrail_tool_calls,
-            guardrail_name=guardrail_name,
+            guardrail_name=guardrail_to_apply.guardrail_name or "unknown",
         )
 
+    @staticmethod
+    def _rebuild_ended_stream_per_choice(
+        responses_so_far: Sequence["ModelResponseStream"],
+        litellm_logging_obj: "LiteLLMLoggingObj | None",
+    ) -> "ModelResponse":
+        """``stream_chunk_builder`` folds every choice of a stream into one index-0
+        choice, so the stream is rebuilt one choice index at a time (every chunk
+        kept, its choices narrowed to that index, so usage-only chunks still
+        count) and the rebuilt choices are stitched into one response, each
+        carrying the index the stream gave it."""
+        choice_indices: Final = tuple(
+            sorted(frozenset(choice.index for response in responses_so_far for choice in response.choices))
+        )
+        rebuilt_by_index: Final = tuple(
+            (
+                index,
+                cast(
+                    ModelResponse,
+                    stream_chunk_builder(
+                        chunks=[  # mutable-ok: callee takes a list
+                            OpenAIChatCompletionsHandler._narrowed_to_choice(response, index)
+                            for response in responses_so_far
+                        ],
+                        logging_obj=litellm_logging_obj,
+                    ),
+                ),
+            )
+            for index in choice_indices
+        )
+        (_, base_response), *_ = rebuilt_by_index
+        stitched_choices: Final = [  # mutable-ok: choices is a List field; a tuple there breaks model_dump round-trips
+            rebuilt.choices[0].model_copy(update=MappingProxyType({"index": index}))
+            for index, rebuilt in rebuilt_by_index
+        ]
+        return base_response.model_copy(update=MappingProxyType({"choices": stitched_choices}))
+
+    @staticmethod
+    def _narrowed_to_choice(response: "ModelResponseStream", index: int) -> "ModelResponseStream":
+        narrowed: Final = [choice for choice in response.choices if choice.index == index]  # mutable-ok: List field
+        return response.model_copy(update=MappingProxyType({"choices": narrowed}))
+
     def build_stream_error_items(
         self,
         exc: "HTTPException",
@@ -1058,39 +1095,28 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         responses_so_far: list["ModelResponseStream"],  # mutable-ok: rewrites the caller's buffered chunks in place
         guardrailed_response: "ModelResponse",
         pre_guardrail_texts: tuple[str | None, ...],
-        guardrail_name: str,
     ) -> None:
         """Write ended-stream guardrail text rewrites back across the buffered
-        chunks: the full rewritten text lands in the choice's first
-        content-carrying chunk and the rest are blanked, the same shape the
-        in-flight write-back uses. Chunks carrying only finish_reason or usage
-        stay untouched. A rewrite on a stream carrying more than one distinct
-        choice index is reported as undeliverable, so the pipeline executor
-        discards it and releases the original chunks."""
+        chunks, one rewrite per rebuilt choice index: the full rewritten text
+        lands in that choice's first content-carrying chunk and the rest are
+        blanked, the same shape the in-flight write-back uses. Chunks carrying
+        only finish_reason or usage stay untouched."""
         post_guardrail_texts: Final = self._string_choice_contents(guardrailed_response)
-        changed: Final = tuple(
-            after
-            for before, after in zip(pre_guardrail_texts, post_guardrail_texts)
-            if before is not None and after is not None and after != before
+        rewrites_by_choice: Final = MappingProxyType(
+            {
+                choice.index: after
+                for choice, before, after in zip(
+                    guardrailed_response.choices, pre_guardrail_texts, post_guardrail_texts
+                )
+                if before is not None and after is not None and after != before
+            }
         )
-        if not changed:
+        if not rewrites_by_choice:
             return
-        stream_choice_indices: Final = frozenset(
-            choice.index for response in responses_so_far for choice in response.choices
-        )
-        if len(stream_choice_indices) != 1:
-            # stream_chunk_builder collapses every choice into one index-0
-            # choice, so a rewrite of the rebuilt response cannot be attributed
-            # back to a single choice on an n>1 stream: report it undeliverable
-            # rather than deliver the rewrite on the wrong choice
-            from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
-
-            raise UndeliverableStreamRewrite(guardrail_name)
-        target_choice_index: Final = next(iter(stream_choice_indices))
         await self._apply_guardrail_responses_to_output_streaming(
             responses=responses_so_far,
-            guardrailed_texts=list(changed),  # mutable-ok: callee takes lists
-            task_mappings=[(target_choice_index, None) for _ in changed],  # mutable-ok: callee takes lists
+            guardrailed_texts=list(rewrites_by_choice.values()),  # mutable-ok: callee takes lists
+            task_mappings=[(index, None) for index in rewrites_by_choice],  # mutable-ok: callee takes lists
         )
 
     @staticmethod