Merge branch 'main' of https://github.com/BerriAI/litellm into litellm_cost_map_background_fetch

# Conflicts:
#	litellm/__init__.py

helm/litellm-helm/README.md

@@ -70,11 +70,18 @@ proxyConfigMap:
 proxy_config:
   general_settings:
     master_key: os.environ/PROXY_MASTER_KEY
+  model_list: []
+```
+
+The chart ships no models, so add them under `proxy_config.model_list` or from the Admin UI with `store_model_in_db: true`
+
+```
+proxy_config:
   model_list:
-    - model_name: gpt-3.5-turbo
+    - model_name: gpt-5.6
       litellm_params:
-        model: gpt-3.5-turbo
-        api_key: eXaMpLeOnLy
+        model: openai/gpt-5.6
+        api_key: os.environ/OPENAI_API_KEY
 ```
 
 #### Example using existing `proxyConfigMap` instead of creating it:

helm/litellm-helm/tests/deployment_tests.yaml

@@ -270,6 +270,32 @@ tests:
       - matchRegex:
           path: data["config.yaml"]
           pattern: '(?m)^\s*health_check_details:\s*false$'
+  - it: should not render any placeholder models when the user sets proxy_config without model_list
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.general_settings.store_model_in_db: true
+    asserts:
+      - notMatchRegex:
+          path: data["config.yaml"]
+          pattern: 'model_name:'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: 'store_model_in_db: true'
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: 'master_key: os.environ/PROXY_MASTER_KEY'
+  - it: should render a user-supplied model_list as the only models
+    template: configmap-litellm.yaml
+    set:
+      proxy_config.model_list:
+        - model_name: my-model
+          litellm_params:
+            model: openai/my-model
+            api_key: os.environ/OPENAI_API_KEY
+    asserts:
+      - matchRegex:
+          path: data["config.yaml"]
+          pattern: 'model_list:\n- litellm_params:\n    api_key: os.environ/OPENAI_API_KEY\n    model: openai/my-model\n  model_name: my-model\n$'
   - it: should allow overriding liveness, readiness, and startup probes
     template: deployment.yaml
     set:

helm/litellm-helm/values.yaml

@@ -180,17 +180,7 @@ proxyConfigMap:
 #  Examples: https://github.com/BerriAI/litellm/tree/main/litellm/proxy/example_config_yaml
 #  Reference: https://docs.litellm.ai/docs/proxy/configs
 proxy_config:
-  model_list:
-    # At least one model must exist for the proxy to start.
-    - model_name: gpt-3.5-turbo
-      litellm_params:
-        model: gpt-3.5-turbo
-        api_key: eXaMpLeOnLy
-    - model_name: fake-openai-endpoint
-      litellm_params:
-        model: openai/fake
-        api_key: fake-key
-        api_base: https://exampleopenaiendpoint-production.up.railway.app/
+  model_list: []
   general_settings:
     master_key: os.environ/PROXY_MASTER_KEY
 

litellm-rust/Cargo.lock

@@ -4083,6 +4083,7 @@ dependencies = [
 name = "litellm-inference-responses"
 version = "0.1.0"
 dependencies = [
+ "base64 0.22.1",
  "bytes",
  "futures-util",
  "litellm-auth",

litellm-rust/crates/inference-chat/src/common_utils.rs

@@ -6,9 +6,11 @@ use litellm_llms::{
     bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
     openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG,
 };
+use litellm_llms_types::formats::chat_completions::ChatMessage;
 use serde_json::{Map, Value};
 
 use super::Error;
+use crate::types::{ChatCompletionsCall, ResolvedChatCompletionsRequest};
 
 const HEADER_CONTEXT: &str = "chat completions";
 
@@ -42,3 +44,58 @@ pub(super) fn string_headers(
 ) -> Result<Vec<(String, String)>, Error> {
     shared_string_headers(HEADER_CONTEXT, extra_headers).map_err(Error::from)
 }
+
+pub(super) fn resolve_request(
+    call: ChatCompletionsCall,
+    config: &'static dyn BaseConfig,
+) -> Result<ResolvedChatCompletionsRequest, Error> {
+    let messages = serde_json::from_value::<Vec<ChatMessage>>(call.messages).map_err(|err| {
+        Error::InvalidRequest(litellm_llms::ErrorDetail::invalid(
+            "chat completions messages",
+            err,
+        ))
+    })?;
+    if messages.is_empty() {
+        return Err(Error::InvalidRequest(
+            "chat completions requires at least one message".into(),
+        ));
+    }
+    if let Some(reason) = config.unsupported_reason(&messages, &call.optional_params) {
+        return Err(Error::Unsupported(reason.0));
+    }
+    Ok(ResolvedChatCompletionsRequest {
+        messages,
+        optional_params: call.optional_params,
+        api_key: call.api_key,
+        api_base: call.api_base,
+        extra_headers: call.extra_headers,
+        timeout: call.timeout,
+    })
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::chat_completions_provider;
+    use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+
+    #[rstest]
+    #[case::anthropic(LlmProviders::Anthropic, ("anthropic-version", "2023-06-01"))]
+    #[case::bedrock(LlmProviders::Bedrock, ("Content-Type", "application/json"))]
+    fn resolves_supported_provider_configs(
+        #[case] provider: LlmProviders,
+        #[case] expected_header: (&str, &str),
+    ) {
+        let config = chat_completions_provider(provider)
+            .expect("provider has a chat completions config")
+            .config();
+        assert!(config.default_headers().contains(&expected_header));
+    }
+
+    #[rstest]
+    #[case::openai(LlmProviders::Openai)]
+    fn rejects_providers_without_a_chat_config(#[case] provider: LlmProviders) {
+        assert!(chat_completions_provider(provider).is_none());
+    }
+}

litellm-rust/crates/inference-chat/src/handler.rs

@@ -3,44 +3,66 @@ use std::time::Duration;
 
 use litellm_auth::AuthServices;
 use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
-use litellm_http::{Client, outbound::OutboundRequest, request::truncate_error_body};
+use litellm_http::{
+    Client,
+    outbound::OutboundRequest,
+    request::{truncate_error_body, with_default_headers},
+};
 use litellm_llms::base_llm::{
-    auth::{Authenticated, resolve_auth},
+    auth::{Authenticated, ValidatedEnvironment, resolve_auth},
+    chat::transformation::BaseConfig,
     chat::transformation::ProviderChatResponseData,
 };
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
+use litellm_secrets::source::Secrets;
 use serde_json::Value;
 
 use super::Error;
-use crate::{constants::CHAT_COMPLETIONS_TIMEOUT_SECS, types::ProviderChatCompletionsRequest};
+use crate::{
+    common_utils::string_headers, constants::CHAT_COMPLETIONS_TIMEOUT_SECS,
+    types::ResolvedChatCompletionsRequest,
+};
 
+#[expect(clippy::too_many_arguments)] // mirrors BaseLLMHTTPHandler.completion, which takes each stage input separately
 pub(super) async fn execute(
     http: &Client,
     auth: &AuthServices,
-    request: ProviderChatCompletionsRequest,
+    config: &'static dyn BaseConfig,
+    provider: litellm_inference::provider::ResolvedProvider<'_>,
+    call: ResolvedChatCompletionsRequest,
+    secrets: Secrets,
     cache: Option<litellm_cache_response::ScopedCache>,
     cache_options: Option<litellm_cache_response::CachePolicy>,
     interceptors: &impl Interceptors<Error>,
     observers: Option<&ObservationSender>,
 ) -> Result<ChatCompletionsResponse, Error> {
-    let ProviderChatCompletionsRequest {
-        model,
-        custom_llm_provider,
-        config,
-        url,
-        body,
-        optional_params,
-        environment,
-        secrets,
-        timeout,
-        api_key,
-    } = request;
+    let validated = config.validate_environment(
+        string_headers(call.extra_headers)?,
+        call.api_key.as_deref(),
+        provider.model,
+        &call.optional_params,
+        &|key| secrets.get(key),
+    )?;
+    let url = config.get_complete_url(
+        call.api_base.as_deref(),
+        provider.model,
+        &call.optional_params,
+        &|key| secrets.get(key),
+    )?;
+    let body = config
+        .transform_request(provider.model, call.messages, call.optional_params.clone())?
+        .body;
+    let environment = ValidatedEnvironment {
+        headers: with_default_headers(validated.headers, config.default_headers()),
+        auth: validated.auth,
+    };
+    let provider_name = <&'static str>::from(provider.provider);
     let context = RequestContext {
-        model: model.clone(),
-        custom_llm_provider,
-        optional_params: Value::Object(optional_params),
+        model: provider.model.to_owned(),
+        custom_llm_provider: provider_name.to_owned(),
+        optional_params: Value::Object(call.optional_params),
         secret_fields: Vec::new(),
-        api_key,
+        api_key: call.api_key.map(litellm_auth::SecretValue::new),
     };
     let authenticated = resolve_auth(auth, environment, &|key| secrets.get(key)).await?;
     let identity = litellm_host::interceptors::ProviderIdentity {
@@ -76,7 +98,7 @@ pub(super) async fn execute(
                 },
                 wire.url,
                 &wire.body,
-                timeout,
+                call.timeout,
             )?;
 
             let response = litellm_inference::outbound::send(outbound, http)
@@ -121,7 +143,7 @@ pub(super) async fn execute(
                 ))
             })?;
             config
-                .transform_response(&model, ProviderChatResponseData { body })
+                .transform_response(provider.model, ProviderChatResponseData { body })
                 .map_err(Error::from)
                 .map_err(as_response_error)
         },
@@ -134,7 +156,7 @@ pub(super) async fn execute(
 ///
 /// A config reports the same variants on either side of the call: a missing
 /// field or an unsupported block can mean "this request cannot be translated"
-/// during prepare and "this response cannot be normalized" here. Only the
+/// before the request is sent and "this response cannot be normalized" here. Only the
 /// second kind has already been billed, and a host that keeps a reference
 /// implementation must not retry those, so collapse them to one variant that
 /// can only mean the provider was already called.
@@ -178,9 +200,10 @@ mod tests {
 
     use super::*;
     use crate::{
-        prepare::{prepare_provider_request, resolve_request},
-        types::ChatCompletionsRequest,
+        common_utils::{chat_completions_provider, resolve_request},
+        types::{ChatCompletionsCall, ResolvedChatCompletionsRequest},
     };
+    use litellm_inference::provider::resolve_llm_provider;
 
     const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
 
@@ -217,22 +240,33 @@ mod tests {
         }
     }
 
-    fn prepared(api_base: &str) -> ProviderChatCompletionsRequest {
-        prepare_provider_request(
-            resolve_request(ChatCompletionsRequest {
-                model: "anthropic/claude-sonnet-4-5",
+    fn prepared(
+        api_base: &str,
+    ) -> (
+        litellm_inference::provider::ResolvedProvider<'static>,
+        &'static dyn BaseConfig,
+        ResolvedChatCompletionsRequest,
+    ) {
+        let provider =
+            resolve_llm_provider("anthropic/claude-sonnet-4-5", None, "chat completions").unwrap();
+        let config = chat_completions_provider(provider.provider)
+            .expect("provider has a chat completions config")
+            .config();
+        let request = resolve_request(
+            ChatCompletionsCall {
+                model: "anthropic/claude-sonnet-4-5".into(),
                 messages: json!([{"role": "user", "content": "hi"}]),
                 optional_params: json!({"max_tokens": 16}).as_object().unwrap().clone(),
-                api_key: Some("sk-test"),
-                api_base: Some(api_base),
+                api_key: Some("sk-test".into()),
+                api_base: Some(api_base.into()),
                 custom_llm_provider: None,
                 extra_headers: None,
                 timeout: None,
-            })
-            .unwrap(),
-            std::sync::Arc::new(|_: &str| None),
+            },
+            config,
         )
-        .unwrap()
+        .unwrap();
+        (provider, config, request)
     }
 
     #[rstest]
@@ -247,10 +281,14 @@ mod tests {
             .await;
         let interceptors = RecordingHooks::default();
 
+        let (provider, config, request) = prepared(&upstream.uri());
         execute(
             &Client::plain_for_test(),
             &AuthServices::default(),
-            prepared(&upstream.uri()),
+            config,
+            provider,
+            request,
+            std::sync::Arc::new(|_: &str| None),
             None,
             None,
             &interceptors,
@@ -288,10 +326,14 @@ mod tests {
             .await;
         let interceptors = RecordingHooks::default();
 
+        let (provider, config, request) = prepared(&upstream.uri());
         let error = execute(
             &Client::plain_for_test(),
             &AuthServices::default(),
-            prepared(&upstream.uri()),
+            config,
+            provider,
+            request,
+            std::sync::Arc::new(|_: &str| None),
             None,
             None,
             &interceptors,

litellm-rust/crates/inference-chat/src/lib.rs

@@ -5,14 +5,13 @@ pub use litellm_inference::RouteError as Error;
 mod common_utils;
 pub mod constants;
 pub(crate) mod handler;
-mod prepare;
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
-use prepare::{prepare_provider_request, resolve_request};
 
 use litellm_auth::AuthServices;
+use litellm_inference::provider::resolve_llm_provider;
 use litellm_secrets::source::SecretSource;
 use std::sync::Arc;
-use types::ChatCompletionsRequest;
+use types::{ChatCompletionsCall, ChatCompletionsRequest};
 
 #[derive(Clone)]
 pub struct ChatCompletionsRoute {
@@ -55,7 +54,7 @@ impl ChatCompletionsRoute {
         } = options.into();
         litellm_host::lifecycle::observe_unary(
             observers.clone(),
-            self.run_call(
+            self.run(
                 request.into(),
                 cache_options,
                 interceptors,
@@ -65,30 +64,52 @@ impl ChatCompletionsRoute {
         .await
     }
 
+    #[tracing::instrument(name = "litellm.route", skip_all, fields(
+        route = "chat_completions",
+        model = %call.model,
+        provider,
+        resolved_model,
+        stream = false,
+        outcome
+    ))]
     async fn run(
         &self,
-        request: ChatCompletionsRequest<'_>,
+        call: ChatCompletionsCall,
         cache_options: Option<litellm_cache_response::CachePolicy>,
         interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
         observers: Option<&ObservationSender>,
     ) -> Result<ChatCompletionsResponse, Error> {
-        let resolved = resolve_request(request)?;
-        let snapshot = self
-            .secrets
-            .resolve(&resolved.config.secret_names())
-            .await?;
-        let prepared = prepare_provider_request(resolved, snapshot)?;
-        litellm_inference::diagnostic::provider(&prepared.model, &prepared.custom_llm_provider);
-        let execute: futures_util::future::BoxFuture<'_, Result<ChatCompletionsResponse, Error>> =
-            Box::pin(handler::execute(
+        litellm_inference::diagnostic::unary(async {
+            let model = call.model.clone();
+            let custom_llm_provider = call.custom_llm_provider.clone();
+            let provider =
+                resolve_llm_provider(&model, custom_llm_provider.as_deref(), "chat completions")?;
+            let config = common_utils::chat_completions_provider(provider.provider)
+                .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?
+                .config();
+            let call = common_utils::resolve_request(call, config)?;
+            litellm_inference::diagnostic::provider(
+                provider.model,
+                <&'static str>::from(provider.provider),
+            );
+            let secrets = self.secrets.resolve(&config.secret_names()).await?;
+            let execute: futures_util::future::BoxFuture<
+                '_,
+                Result<ChatCompletionsResponse, Error>,
+            > = Box::pin(handler::execute(
                 &self.http,
                 &self.auth,
-                prepared,
+                config,
+                provider,
+                call,
+                secrets,
                 self.cache.clone(),
                 cache_options,
                 interceptors,
                 observers,
             ));
-        execute.await
+            execute.await
+        })
+        .await
     }
 }

litellm-rust/crates/inference-chat/src/prepare.rs

@@ -1,638 +0,0 @@
-use litellm_auth::SecretValue;
-use litellm_core_utils::settings::Lookup;
-use litellm_http::request::with_default_headers;
-use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig};
-use litellm_llms_types::formats::chat_completions::ChatMessage;
-use litellm_secrets::source::Secrets;
-use serde_json::Value;
-
-use super::{
-    Error,
-    common_utils::{chat_completions_provider, string_headers},
-};
-use crate::types::{
-    ChatCompletionsRequest, ProviderChatCompletionsRequest, ResolvedChatCompletionsRequest,
-};
-use litellm_inference::provider::resolve_llm_provider;
-
-pub(super) struct ResolvedProvider {
-    pub(super) model: String,
-    pub(super) custom_llm_provider: String,
-    pub(super) config: &'static dyn BaseConfig,
-}
-
-pub(super) fn resolve_provider_config<'a>(
-    model: &'a str,
-    custom_llm_provider: Option<&'a str>,
-) -> Result<ResolvedProvider, Error> {
-    let provider_info = resolve_llm_provider(model, custom_llm_provider, "chat completions")?;
-    let config = chat_completions_provider(provider_info.provider)
-        .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?
-        .config();
-    Ok(ResolvedProvider {
-        model: provider_info.model.to_string(),
-        custom_llm_provider: <&str>::from(provider_info.provider).to_string(),
-        config,
-    })
-}
-
-pub(super) fn parse_messages(messages: Value) -> Result<Vec<ChatMessage>, Error> {
-    serde_json::from_value(messages).map_err(|err| {
-        Error::InvalidRequest(litellm_llms::ErrorDetail::invalid(
-            "chat completions messages",
-            err,
-        ))
-    })
-}
-
-pub(super) fn resolve_request(
-    request: ChatCompletionsRequest<'_>,
-) -> Result<ResolvedChatCompletionsRequest<'_>, Error> {
-    let ResolvedProvider {
-        model,
-        custom_llm_provider,
-        config,
-    } = resolve_provider_config(request.model, request.custom_llm_provider)?;
-    let messages = parse_messages(request.messages)?;
-    if messages.is_empty() {
-        return Err(Error::InvalidRequest(
-            "chat completions requires at least one message".into(),
-        ));
-    }
-    if let Some(reason) = config.unsupported_reason(&messages, &request.optional_params) {
-        return Err(Error::Unsupported(reason.0));
-    }
-    Ok(ResolvedChatCompletionsRequest {
-        model,
-        custom_llm_provider,
-        config,
-        messages,
-        optional_params: request.optional_params,
-        api_key: request.api_key,
-        api_base: request.api_base,
-        extra_headers: request.extra_headers,
-        timeout: request.timeout,
-    })
-}
-
-fn validate_environment(
-    request: &ResolvedChatCompletionsRequest<'_>,
-    model: &str,
-    config: &dyn BaseConfig,
-    secrets: &dyn Lookup,
-) -> Result<ValidatedEnvironment, Error> {
-    let env_lookup = |key: &str| secrets.get(key);
-    let forwarded = string_headers(request.extra_headers.clone())?;
-    let validated = config.validate_environment(
-        forwarded,
-        request.api_key,
-        model,
-        &request.optional_params,
-        &env_lookup,
-    )?;
-    Ok(ValidatedEnvironment {
-        headers: with_default_headers(validated.headers, config.default_headers()),
-        auth: validated.auth,
-    })
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub(super) fn prepare_provider_request(
-    request: ResolvedChatCompletionsRequest<'_>,
-    secrets: Secrets,
-) -> Result<ProviderChatCompletionsRequest, Error> {
-    let environment =
-        validate_environment(&request, &request.model, request.config, secrets.as_ref())?;
-    let model = request.model;
-    let config = request.config;
-    let env_lookup = |key: &str| secrets.get(key);
-    let url = config.get_complete_url(
-        request.api_base,
-        &model,
-        &request.optional_params,
-        &env_lookup,
-    )?;
-    let transformed =
-        config.transform_request(&model, request.messages, request.optional_params.clone())?;
-
-    Ok(ProviderChatCompletionsRequest {
-        model,
-        custom_llm_provider: request.custom_llm_provider,
-        config,
-        url,
-        body: transformed.body,
-        optional_params: request.optional_params,
-        environment,
-        secrets,
-        timeout: request.timeout,
-        api_key: request.api_key.map(|key| SecretValue::new(key.to_string())),
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use litellm_auth::CredentialPlacement;
-    use litellm_llms::base_llm::auth::{AuthScheme, resolve_auth};
-    use serde_json::{Map, Value, json};
-
-    use super::{prepare_provider_request, resolve_request};
-    use crate::{
-        Error,
-        types::{ChatCompletionsRequest, ProviderChatCompletionsRequest},
-    };
-
-    fn prepare_chat_completions_call(
-        request: ChatCompletionsRequest<'_>,
-    ) -> Result<ProviderChatCompletionsRequest, Error> {
-        prepare_provider_request(
-            resolve_request(request)?,
-            std::sync::Arc::new(|_: &str| None),
-        )
-    }
-
-    /// The headers as they go on the wire, credential applied.
-    fn wire_headers(prepared: &ProviderChatCompletionsRequest) -> Vec<(String, String)> {
-        tokio::runtime::Builder::new_current_thread()
-            .build()
-            .unwrap()
-            .block_on(resolve_auth(
-                &litellm_auth::AuthServices::default(),
-                prepared.environment.clone(),
-                &|_| None,
-            ))
-            .unwrap()
-            .headers
-    }
-
-    fn request<'a>(
-        model: &'a str,
-        provider: Option<&'a str>,
-        messages: Value,
-        optional_params: Value,
-    ) -> ChatCompletionsRequest<'a> {
-        ChatCompletionsRequest {
-            model,
-            messages,
-            optional_params: match optional_params {
-                Value::Object(map) => map,
-                other => panic!("params must be an object, got {other}"),
-            },
-            api_key: Some("sk-test"),
-            api_base: None,
-            custom_llm_provider: provider,
-            extra_headers: None,
-            timeout: None,
-        }
-    }
-
-    fn preparation_error(request: ChatCompletionsRequest<'_>) -> Error {
-        prepare_chat_completions_call(request)
-            .err()
-            .expect("request preparation should fail")
-    }
-
-    #[test]
-    fn resolves_the_provider_from_the_model_prefix() {
-        let prepared = prepare_chat_completions_call(request(
-            "anthropic/claude-sonnet-4-5",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"max_tokens": 16}),
-        ))
-        .expect("prepares");
-        assert_eq!(prepared.model, "claude-sonnet-4-5");
-        assert_eq!(prepared.url, "https://api.anthropic.com/v1/messages");
-        assert_eq!(prepared.body["model"], json!("claude-sonnet-4-5"));
-    }
-
-    #[test]
-    fn strips_an_explicit_provider_prefix_from_the_model() {
-        let prepared = prepare_chat_completions_call(request(
-            "anthropic/claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        ))
-        .expect("prepares");
-        assert_eq!(prepared.model, "claude-sonnet-4-5");
-    }
-
-    #[test]
-    fn adds_the_auth_and_default_headers() {
-        let prepared = prepare_chat_completions_call(request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        ))
-        .expect("prepares");
-        assert!(
-            wire_headers(&prepared).contains(&("x-api-key".to_string(), "sk-test".to_string()))
-        );
-        assert!(
-            wire_headers(&prepared)
-                .contains(&("anthropic-version".to_string(), "2023-06-01".to_string()))
-        );
-        assert!(matches!(
-            prepared.environment.auth,
-            AuthScheme::Credential {
-                placement: CredentialPlacement::Header("x-api-key"),
-                ..
-            }
-        ));
-    }
-
-    #[test]
-    fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
-        // Python builds `{**headers, **anthropic_headers}`, so the deployment's key
-        // overwrites a forwarded one. Honouring the caller's would let whoever sends
-        // the request choose the Anthropic principal it bills to.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([(
-            "X-Api-Key".to_string(),
-            json!("sk-caller"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let keys: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
-            .collect();
-        assert_eq!(keys.len(), 1, "got {:?}", headers);
-        assert_eq!(keys[0].1, "sk-test");
-    }
-
-    #[test]
-    fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
-        // Anthropic's `validate_environment` pops `x-api-key` and sets `authorization`
-        // for an OAuth token, so re-adding the key here would put the credential into
-        // a header the host removed on purpose.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([
-            (
-                "Authorization".to_string(),
-                json!("Bearer sk-ant-oat01-token"),
-            ),
-            ("X-Api-Key".to_string(), json!("sk-caller")),
-        ]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        assert!(
-            !wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("x-api-key") && value == "sk-test"),
-            "the resolved key must not be applied over an OAuth bearer, got {:?}",
-            wire_headers(&prepared)
-        );
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer sk-ant-oat01-token")
-        );
-    }
-
-    #[test]
-    fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
-        // Only an OAuth bearer replaces the credential. Python sends the deployment's
-        // `x-api-key` alongside any other forwarded `authorization`, so deferring on
-        // the mere presence of that header would drop the deployment's auth.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([
-            ("Authorization".to_string(), json!("Bearer unrelated")),
-            ("X-Api-Key".to_string(), json!("sk-caller")),
-        ]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let keys: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
-            .collect();
-        assert_eq!(keys.len(), 1, "got {:?}", headers);
-        assert_eq!(keys[0].1, "sk-test");
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer unrelated"),
-            "the unrelated authorization must survive, got {:?}",
-            wire_headers(&prepared)
-        );
-    }
-
-    #[rstest::rstest]
-    fn rejects_empty_messages_before_resolving_credentials() {
-        let call = ChatCompletionsRequest {
-            api_key: None,
-            ..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({}))
-        };
-        assert!(matches!(preparation_error(call), Error::InvalidRequest(_)));
-    }
-
-    #[test]
-    fn rejects_an_unknown_provider() {
-        assert_eq!(
-            preparation_error(request(
-                "openai/gpt-4o",
-                None,
-                json!([{"role": "user", "content": "hi"}]),
-                json!({}),
-            )),
-            Error::InvalidProvider("openai".to_string())
-        );
-    }
-
-    #[test]
-    fn rejects_a_model_with_no_resolvable_provider() {
-        assert!(matches!(
-            preparation_error(request(
-                "claude-sonnet-4-5",
-                None,
-                json!([{"role": "user", "content": "hi"}]),
-                json!({}),
-            )),
-            Error::InvalidProvider(_)
-        ));
-    }
-
-    #[test]
-    fn rejects_an_empty_or_malformed_message_list() {
-        assert_eq!(
-            preparation_error(request(
-                "anthropic/claude-sonnet-4-5",
-                None,
-                json!([]),
-                json!({}),
-            )),
-            Error::InvalidRequest(
-                "chat completions requires at least one message"
-                    .to_string()
-                    .into()
-            )
-        );
-        assert!(matches!(
-            preparation_error(request(
-                "anthropic/claude-sonnet-4-5",
-                None,
-                json!("not a list"),
-                json!({}),
-            )),
-            Error::InvalidRequest(_)
-        ));
-    }
-
-    #[test]
-    fn rejects_non_string_extra_headers() {
-        let mut call = request(
-            "anthropic/claude-sonnet-4-5",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([("x-trace".to_string(), json!(7))]));
-        assert_eq!(
-            preparation_error(call),
-            Error::Headers(litellm_http::request::HeaderError {
-                context: "chat completions",
-                name: "x-trace".to_string(),
-                actual: "number",
-            })
-        );
-    }
-
-    #[test]
-    fn prepares_a_bedrock_call_without_resolving_credentials() {
-        let mut call = request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"maxTokens": 16}),
-        );
-        call.api_key = None;
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        assert_eq!(
-            prepared.url,
-            "https://bedrock-runtime.us-east-1.amazonaws.com/model/anthropic.claude-v2/converse"
-        );
-        assert!(matches!(
-            &prepared.environment.auth,
-            AuthScheme::AwsSigV4 { region, service: "bedrock", .. } if region == "us-east-1"
-        ));
-        // SigV4 signs the serialized body, so prepare must not have added an
-        // Authorization header; the signer does it over the bytes sent.
-        assert!(
-            !prepared
-                .environment
-                .headers
-                .iter()
-                .any(|(name, _)| name.eq_ignore_ascii_case("authorization"))
-        );
-        assert_eq!(prepared.body["inferenceConfig"], json!({"maxTokens": 16}));
-    }
-
-    #[tokio::test]
-    async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
-        // Python signs only the AWS header set and reattaches the rest, so a header
-        // the caller forwarded rides along without joining the canonical request.
-        // Signing it makes Converse 403 on a deployment that works on Python.
-        let mut call = request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({
-                "maxTokens": 16,
-                "aws_access_key_id": "AKIDEXAMPLE",
-                "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
-            }),
-        );
-        // A key would resolve to a bearer token and never reach the signer.
-        call.api_key = None;
-        call.extra_headers = Some(Map::from_iter([(
-            "x-request-id".to_string(),
-            json!("abc-123"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let authenticated = resolve_auth(
-            &litellm_auth::AuthServices::default(),
-            prepared.environment,
-            &|_| None,
-        )
-        .await
-        .expect("resolves");
-        let signed = crate::handler::outbound_request(
-            authenticated,
-            prepared.url,
-            &prepared.body,
-            prepared.timeout,
-        )
-        .expect("signs");
-
-        let authorization = signed
-            .header("authorization")
-            .expect("carries an authorization header")
-            .to_string();
-        assert!(
-            authorization.starts_with("AWS4-HMAC-SHA256"),
-            "expected a SigV4 signature, got {authorization}"
-        );
-        assert!(
-            !authorization.contains("x-request-id"),
-            "forwarded header reached SignedHeaders: {authorization}"
-        );
-        // It still goes on the wire, it is just not part of the signature.
-        assert!(
-            signed
-                .headers()
-                .iter()
-                .any(|(name, value)| name == "x-request-id" && value == "abc-123"),
-            "forwarded header was dropped instead of reattached"
-        );
-    }
-
-    #[rstest::rstest]
-    #[case::authorization("Authorization")]
-    #[case::amz_date("x-amz-date")]
-    #[case::security_token("x-amz-security-token")]
-    #[case::date("Date")]
-    #[tokio::test]
-    async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
-        let call = ChatCompletionsRequest {
-            api_key: None,
-            extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])),
-            ..request(
-                "bedrock/us-east-1/anthropic.claude-v2",
-                None,
-                json!([{"role": "user", "content": "hi"}]),
-                json!({
-                    "maxTokens": 16,
-                    "aws_access_key_id": "AKIDEXAMPLE",
-                    "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
-                }),
-            )
-        };
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let authenticated = resolve_auth(
-            &litellm_auth::AuthServices::default(),
-            prepared.environment,
-            &|_| None,
-        )
-        .await
-        .expect("resolves");
-        let error = crate::handler::outbound_request(
-            authenticated,
-            prepared.url,
-            &prepared.body,
-            prepared.timeout,
-        )
-        .expect_err("conflicting signing headers must fail");
-        assert!(
-            matches!(error, Error::Unsupported(_)),
-            "{forwarded} returned {error:?}"
-        );
-    }
-
-    #[test]
-    fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
-        // `get_request_headers` assigns `headers["Authorization"]` unconditionally
-        // once a bearer token resolves, so the deployment's identity wins on
-        // Python. Keeping the caller's would authorize and bill the call as a
-        // different principal, and only when the deployment carries `rust: true`.
-        let mut call = request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"maxTokens": 16}),
-        );
-        call.extra_headers = Some(Map::from_iter([(
-            "Authorization".to_string(),
-            json!("Bearer caller-supplied"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let authorizations: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("authorization"))
-            .map(|(_, value)| value.as_str())
-            .collect();
-        assert_eq!(
-            authorizations,
-            vec!["Bearer sk-test"],
-            "the deployment token must be the only authorization on the wire"
-        );
-    }
-
-    #[test]
-    fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
-        // The opposite precedence, and deliberate: Anthropic's own transform
-        // honours a forwarded OAuth bearer, so the Bedrock fix above must not be
-        // generalized into a rule that the configured key always wins.
-        //
-        // An OAuth bearer is the whole of that exception. This forwarded a plain
-        // `x-api-key` until round 17, which read as the same claim and was not:
-        // Python overwrites a forwarded `x-api-key` with the deployment's.
-        let mut call = request(
-            "claude-sonnet-4-5",
-            Some("anthropic"),
-            json!([{"role": "user", "content": "hi"}]),
-            json!({}),
-        );
-        call.extra_headers = Some(Map::from_iter([(
-            "authorization".to_string(),
-            json!("Bearer sk-ant-oat01-forwarded"),
-        )]));
-        let prepared = prepare_chat_completions_call(call).expect("prepares");
-        let headers = wire_headers(&prepared);
-        let keys: Vec<_> = headers
-            .iter()
-            .filter(|(name, _)| name.eq_ignore_ascii_case("x-api-key"))
-            .map(|(_, value)| value.as_str())
-            .collect();
-        assert!(keys.is_empty(), "got {:?}", headers);
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer sk-ant-oat01-forwarded")
-        );
-    }
-
-    #[test]
-    fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
-        // The configured bearer identity has its own account and quota boundary,
-        // so a request carrying one must not be signed as whatever principal the
-        // host's AWS credentials resolve to.
-        let prepared = prepare_chat_completions_call(request(
-            "bedrock/us-east-1/anthropic.claude-v2",
-            None,
-            json!([{"role": "user", "content": "hi"}]),
-            json!({"maxTokens": 16}),
-        ))
-        .expect("prepares");
-        assert!(matches!(
-            &prepared.environment.auth,
-            AuthScheme::Credential { placement: CredentialPlacement::Bearer, secret }
-                if secret.expose() == "sk-test"
-        ));
-        assert!(
-            wire_headers(&prepared)
-                .iter()
-                .any(|(name, value)| name.eq_ignore_ascii_case("authorization")
-                    && value == "Bearer sk-test"),
-            "prepare did not carry the bearer token"
-        );
-    }
-}

litellm-rust/crates/inference-chat/src/route.rs

@@ -1,4 +1,3 @@
-use litellm_host::observation::ObservationSender;
 use std::convert::Infallible;
 
 use litellm_host::{
@@ -7,10 +6,7 @@ use litellm_host::{
 };
 use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse;
 
-use super::{
-    ChatCompletionsRoute, Error,
-    types::{ChatCompletionsCall, ChatCompletionsRequest},
-};
+use super::{ChatCompletionsRoute, Error, types::ChatCompletionsCall};
 
 pub struct ChatCompletions;
 
@@ -37,44 +33,12 @@ impl ChatCompletionsRoute {
             call,
             observers,
             move |call, _, interceptors, observers| async move {
-                self.run_call(call, cache_options, &interceptors, observers.as_ref())
+                self.run(call, cache_options, &interceptors, observers.as_ref())
                     .await
                     .map(CallOutput::Complete)
             },
         )
     }
-
-    #[tracing::instrument(name = "litellm.route", skip_all, fields(
-        route = "chat_completions",
-        model = %call.model,
-        provider,
-        resolved_model,
-        stream = false,
-        outcome
-    ))]
-    pub(super) async fn run_call(
-        &self,
-        call: ChatCompletionsCall,
-        cache_options: Option<litellm_cache_response::CachePolicy>,
-        interceptors: &impl litellm_host::interceptors::Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<ChatCompletionsResponse, Error> {
-        litellm_inference::diagnostic::unary(async {
-            let request = ChatCompletionsRequest {
-                model: &call.model,
-                messages: call.messages,
-                optional_params: call.optional_params,
-                api_key: call.api_key.as_deref(),
-                api_base: call.api_base.as_deref(),
-                custom_llm_provider: call.custom_llm_provider.as_deref(),
-                extra_headers: call.extra_headers,
-                timeout: call.timeout,
-            };
-            self.run(request, cache_options, interceptors, observers)
-                .await
-        })
-        .await
-    }
 }
 
 impl litellm_inference::caching::Cachable for ChatCompletions {

litellm-rust/crates/inference-chat/src/types.rs

@@ -1,8 +1,5 @@
-use litellm_secrets::source::Secrets;
 use std::time::Duration;
 
-use litellm_auth::SecretValue;
-use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig};
 use litellm_llms_types::formats::chat_completions::ChatMessage;
 use serde_json::{Map, Value};
 
@@ -49,31 +46,11 @@ impl From<ChatCompletionsRequest<'_>> for ChatCompletionsCall {
     }
 }
 
-pub struct ResolvedChatCompletionsRequest<'a> {
-    pub model: String,
-    pub custom_llm_provider: String,
-    pub config: &'static dyn BaseConfig,
+pub struct ResolvedChatCompletionsRequest {
     pub messages: Vec<ChatMessage>,
     pub optional_params: Map<String, Value>,
-    pub api_key: Option<&'a str>,
-    pub api_base: Option<&'a str>,
+    pub api_key: Option<String>,
+    pub api_base: Option<String>,
     pub extra_headers: Option<Map<String, Value>>,
     pub timeout: Option<Duration>,
 }
-
-pub struct ProviderChatCompletionsRequest {
-    pub model: String,
-    pub custom_llm_provider: String,
-    pub config: &'static dyn BaseConfig,
-    pub url: String,
-    pub body: Value,
-    /// The route's parameters before the provider transformation, reported to the host
-    /// beside the wire request.
-    pub optional_params: Map<String, Value>,
-    /// The forwarded and default headers plus how the call authenticates; the credential
-    /// itself is applied when the request is sent.
-    pub environment: ValidatedEnvironment,
-    pub secrets: Secrets,
-    pub timeout: Option<Duration>,
-    pub api_key: Option<SecretValue>,
-}

litellm-rust/crates/inference-chat/tests/request.rs

@@ -0,0 +1,439 @@
+use std::time::Duration;
+
+use litellm_inference_chat::{Error, types::ChatCompletionsRequest};
+use rstest::rstest;
+use serde_json::{Map, Value, json};
+use wiremock::ResponseTemplate;
+
+mod support;
+use support::*;
+
+const ANTHROPIC_MESSAGE: &str = r#"{"id":"msg_1","type":"message","role":"assistant","model":"claude-sonnet-4-5","content":[{"type":"text","text":"hello"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":11,"output_tokens":4}}"#;
+
+fn request<'a>(
+    model: &'a str,
+    provider: Option<&'a str>,
+    messages: Value,
+    optional_params: Value,
+) -> ChatCompletionsRequest<'a> {
+    let Value::Object(optional_params) = optional_params else {
+        panic!("params must be an object");
+    };
+    ChatCompletionsRequest {
+        model,
+        messages,
+        optional_params,
+        api_key: Some("sk-test"),
+        api_base: None,
+        custom_llm_provider: provider,
+        extra_headers: None,
+        timeout: Some(Duration::from_secs(10)),
+    }
+}
+
+async fn complete(
+    request: ChatCompletionsRequest<'_>,
+) -> Result<litellm_llms_types::formats::chat_completions::ChatCompletionsResponse, Error> {
+    chat_completions_route().execute(request, &(), None).await
+}
+
+fn object(value: Value) -> Map<String, Value> {
+    let Value::Object(map) = value else {
+        panic!("expected an object");
+    };
+    map
+}
+
+fn anthropic_response() -> ResponseTemplate {
+    ResponseTemplate::new(200).set_body_raw(ANTHROPIC_MESSAGE, "application/json")
+}
+
+fn bedrock_response() -> ResponseTemplate {
+    json_response(json!({
+        "output": {"message": {"role": "assistant", "content": [{"text": "hello"}]}},
+        "stopReason": "end_turn",
+        "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}
+    }))
+}
+
+#[rstest]
+#[tokio::test]
+async fn resolves_the_provider_from_the_model_prefix() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "anthropic/claude-sonnet-4-5",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"max_tokens": 16}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.url.path(), "/v1/messages");
+    assert_eq!(sent.json()["model"], "claude-sonnet-4-5");
+}
+
+#[rstest]
+#[tokio::test]
+async fn strips_an_explicit_provider_prefix_from_the_model() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "anthropic/claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.json()["model"],
+        "claude-sonnet-4-5"
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn adds_the_auth_and_default_headers() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values("x-api-key"), ["sk-test"]);
+    assert_eq!(sent.header("anthropic-version"), Some("2023-06-01"));
+}
+
+#[rstest]
+#[tokio::test]
+async fn the_deployment_credential_replaces_a_caller_supplied_auth_header() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(object(json!({"X-Api-Key": "sk-caller"}))),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.header_values("x-api-key"),
+        ["sk-test"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_forwarded_authorization_header_suppresses_the_resolved_api_key_header() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(object(json!({
+            "Authorization": "Bearer sk-ant-oat01-token",
+            "X-Api-Key": "sk-caller"
+        }))),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values("x-api-key"), ["sk-caller"]);
+    assert_eq!(
+        sent.header_values("authorization"),
+        ["Bearer sk-ant-oat01-token"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_unrelated_forwarded_authorization_does_not_defer_the_resolved_key() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(object(json!({
+            "Authorization": "Bearer unrelated",
+            "X-Api-Key": "sk-caller"
+        }))),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert_eq!(sent.header_values("x-api-key"), ["sk-test"]);
+    assert_eq!(sent.header_values("authorization"), ["Bearer unrelated"]);
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_empty_messages_before_resolving_credentials() {
+    let error = complete(ChatCompletionsRequest {
+        api_key: None,
+        ..request("claude-sonnet-4-5", Some("anthropic"), json!([]), json!({}))
+    })
+    .await
+    .expect_err("empty messages are rejected");
+    assert!(matches!(error, Error::InvalidRequest(_)));
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_an_unknown_provider() {
+    assert_eq!(
+        complete(request(
+            "openai/gpt-4o",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        ))
+        .await
+        .unwrap_err(),
+        Error::InvalidProvider("openai".into())
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_a_model_with_no_resolvable_provider() {
+    assert!(matches!(
+        complete(request(
+            "claude-sonnet-4-5",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        ))
+        .await,
+        Err(Error::InvalidProvider(_))
+    ));
+}
+
+#[rstest]
+#[case::empty(json!([]), Some("chat completions requires at least one message"))]
+#[case::malformed(json!("not a list"), None)]
+#[tokio::test]
+async fn rejects_an_empty_or_malformed_message_list(
+    #[case] messages: Value,
+    #[case] expected: Option<&str>,
+) {
+    let error = complete(request(
+        "anthropic/claude-sonnet-4-5",
+        None,
+        messages,
+        json!({}),
+    ))
+    .await
+    .unwrap_err();
+    if let Some(expected) = expected {
+        assert_eq!(error, Error::InvalidRequest(expected.to_string().into()));
+    } else {
+        assert!(matches!(error, Error::InvalidRequest(_)));
+    }
+}
+
+#[rstest]
+#[tokio::test]
+async fn rejects_non_string_extra_headers() {
+    assert_eq!(
+        complete(ChatCompletionsRequest {
+            extra_headers: Some(Map::from_iter([("x-trace".to_string(), json!(7))])),
+            ..request(
+                "anthropic/claude-sonnet-4-5",
+                None,
+                json!([{"role": "user", "content": "hi"}]),
+                json!({}),
+            )
+        })
+        .await
+        .unwrap_err(),
+        Error::Headers(litellm_http::request::HeaderError {
+            context: "chat completions",
+            name: "x-trace".to_string(),
+            actual: "number",
+        })
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_forwarded_client_header_does_not_enter_the_bedrock_signature() {
+    let upstream = upstream([bedrock_response()]).await;
+    let base = upstream.uri();
+    let mut call = request(
+        "bedrock/us-east-1/anthropic.claude-v2",
+        None,
+        json!([{"role": "user", "content": "hi"}]),
+        json!({
+            "maxTokens": 16,
+            "aws_access_key_id": "AKIDEXAMPLE",
+            "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
+        }),
+    );
+    call.api_key = None;
+    call.api_base = Some(&base);
+    call.extra_headers = Some(Map::from_iter([(
+        "x-request-id".to_string(),
+        json!("abc-123"),
+    )]));
+    complete(call).await.expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    let authorization = sent.header("authorization").expect("signed request");
+    assert!(authorization.starts_with("AWS4-HMAC-SHA256"));
+    assert!(!authorization.contains("x-request-id"));
+    assert_eq!(sent.header("x-request-id"), Some("abc-123"));
+    assert_eq!(sent.url.path(), "/model/anthropic.claude-v2/converse");
+    let body: Value = serde_json::from_slice(&sent.body).unwrap();
+    assert_eq!(body["inferenceConfig"], json!({"maxTokens": 16}));
+    assert_eq!(
+        body["messages"],
+        json!([{"role": "user", "content": [{"text": "hi"}]}])
+    );
+}
+
+#[rstest]
+#[case::authorization("Authorization")]
+#[case::amz_date("x-amz-date")]
+#[case::security_token("x-amz-security-token")]
+#[case::date("Date")]
+#[tokio::test]
+async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) {
+    let upstream = wiremock::MockServer::start().await;
+    let base = upstream.uri();
+    let call = ChatCompletionsRequest {
+        api_key: None,
+        api_base: Some(&base),
+        extra_headers: Some(Map::from_iter([(forwarded.to_string(), json!("forged"))])),
+        ..request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({
+                "maxTokens": 16,
+                "aws_access_key_id": "AKIDEXAMPLE",
+                "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY"
+            }),
+        )
+    };
+    let error = complete(call)
+        .await
+        .expect_err("conflicting signing headers fail");
+    assert!(matches!(error, Error::Unsupported(_)), "{error:?}");
+    assert!(received(&upstream).await.is_empty());
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_bedrock_deployment_bearer_outranks_a_forwarded_authorization() {
+    let upstream = upstream([bedrock_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(Map::from_iter([(
+            "Authorization".to_string(),
+            json!("Bearer caller-supplied"),
+        )])),
+        ..request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"maxTokens": 16}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.header_values("authorization"),
+        ["Bearer sk-test"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn an_anthropic_forwarded_oauth_bearer_still_outranks_the_resolved_key() {
+    let upstream = upstream([anthropic_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        extra_headers: Some(Map::from_iter([(
+            "authorization".to_string(),
+            json!("Bearer sk-ant-oat01-forwarded"),
+        )])),
+        ..request(
+            "claude-sonnet-4-5",
+            Some("anthropic"),
+            json!([{"role": "user", "content": "hi"}]),
+            json!({}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    let sent = only_request(&upstream).await;
+    assert!(sent.header_values("x-api-key").is_empty());
+    assert_eq!(
+        sent.header_values("authorization"),
+        ["Bearer sk-ant-oat01-forwarded"]
+    );
+}
+
+#[rstest]
+#[tokio::test]
+async fn a_bedrock_api_key_is_sent_as_a_bearer_token_instead_of_being_signed() {
+    let upstream = upstream([bedrock_response()]).await;
+    let base = upstream.uri();
+    complete(ChatCompletionsRequest {
+        api_base: Some(&base),
+        ..request(
+            "bedrock/us-east-1/anthropic.claude-v2",
+            None,
+            json!([{"role": "user", "content": "hi"}]),
+            json!({"maxTokens": 16}),
+        )
+    })
+    .await
+    .expect("call succeeds");
+
+    assert_eq!(
+        only_request(&upstream).await.header_values("authorization"),
+        ["Bearer sk-test"]
+    );
+}

litellm-rust/crates/inference-messages/AGENTS.md

@@ -7,8 +7,8 @@
   - delegate authentication policy, beta selection, payload rewriting, and response interpretation to those adapters
   - keep provider policy out of request preparation and transport handlers
   - calling a concrete provider helper for every provider is still a policy dependency
-- Route types such as `MessagesCall`, prepared requests, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
+- Route types such as `MessagesCall`, provider calls, and response wrappers containing live streams describe execution. Reuse the shared Messages payload types inside them instead of defining another request or response schema here
 - The route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks
-- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `src/lib.rs` explicitly sequences cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage
+- Per-call dependencies are grouped in `litellm_inference::context::CallContext`. `handler::execute` explicitly sequences the provider call build in Python's handler order, cache lookup, provider execution, result acceptance (`CallContext::result_ready`), and cache storage
 - Preserve the order of validation, normalization, caller-requested parameter removal, and provider transformation when that order affects observable behavior
 - Test provider dispatch, auth precedence, header handling, transformations, and responses through behavior, not source structure

litellm-rust/crates/inference-messages/src/common_utils.rs

@@ -82,17 +82,17 @@ mod tests {
     use litellm_core_utils::get_llm_provider_logic::LlmProviders;
 
     #[rstest]
-    #[case::tencent("tencent")]
-    #[case::anthropic("anthropic")]
-    #[case::azure_ai("azure_ai")]
-    #[case::bedrock("bedrock")]
-    #[case::deepseek("deepseek")]
-    #[case::edenai("edenai")]
-    #[case::minimax("minimax")]
-    #[case::vertex_ai("vertex_ai")]
-    fn provider_keeps_its_python_name(#[case] name: &str) {
-        let provider =
-            messages_provider(name.parse::<LlmProviders>().unwrap(), "claude-sonnet-4-5").unwrap();
+    #[case::tencent(LlmProviders::Tencent, "tencent")]
+    #[case::anthropic(LlmProviders::Anthropic, "anthropic")]
+    #[case::azure_ai(LlmProviders::AzureAi, "azure_ai")]
+    #[case::bedrock(LlmProviders::Bedrock, "bedrock")]
+    #[case::deepseek(LlmProviders::Deepseek, "deepseek")]
+    #[case::edenai(LlmProviders::Edenai, "edenai")]
+    #[case::minimax(LlmProviders::Minimax, "minimax")]
+    #[case::vertex_ai(LlmProviders::VertexAi, "vertex_ai")]
+    #[case::openrouter(LlmProviders::Openrouter, "openrouter")]
+    fn provider_keeps_its_python_name(#[case] provider: LlmProviders, #[case] name: &str) {
+        let provider = messages_provider(provider, "claude-sonnet-4-5").unwrap();
         assert_eq!(provider.as_str(), name);
     }
 

litellm-rust/crates/inference-messages/src/handler.rs

@@ -1,148 +1,271 @@
 use std::time::Duration;
 
 use bytes::Bytes;
-use futures_util::{StreamExt, TryStreamExt, stream::BoxStream};
-use litellm_host::interceptors::{Interceptors, ProviderIdentity, RequestContext, WireRequest};
-use litellm_http::transport::Error as TransportError;
+use futures_util::{FutureExt, StreamExt, TryStreamExt, stream::BoxStream};
+use litellm_auth::{AuthServices, SecretValue};
+use litellm_core_utils::{
+    dot_notation_indexing::delete_nested_value,
+    get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
+};
+use litellm_host::interceptors::{
+    Cost, ExecutionFacts, Interceptors, ProviderIdentity, RequestContext, ResultSource, WireRequest,
+};
+use litellm_http::{request::with_default_headers, transport::Error as TransportError};
 use litellm_llms::base_llm::{
-    auth::{Authenticated, resolve_auth},
+    auth::{Authenticated, ValidatedEnvironment, resolve_auth},
     messages::{
+        context::MessagesTransformContext,
         streaming::{self, ByteStream, StreamDecoder},
         transformation::BaseMessagesConfig,
     },
 };
-use litellm_llms_types::formats::messages::MessagesResponse;
+use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use litellm_secrets::source::Secrets;
 use litellm_tracing::ByteChunk;
 use serde_json::Value;
 
 use super::{
-    Error, MessagesCallResponse, MessagesRoute, common_utils::truncate_error_body,
-    prepare::ProviderMessagesRequest,
+    Error, MessagesCall, MessagesCallResponse, MessagesRoute,
+    common_utils::{MessagesProvider, string_headers, truncate_error_body},
+    constants::MESSAGES_TIMEOUT_SECS,
+    route::Messages,
+    types::invalid_request,
 };
-use crate::constants::MESSAGES_TIMEOUT_SECS;
-use litellm_inference::{context::CallContext, outbound::outbound_request};
+use litellm_inference::{caching::CallCache, context::CallContext, outbound::outbound_request};
 
-pub(super) struct ProviderCall {
-    pub identity: ProviderIdentity,
-    pub wire: WireRequest,
-    pub(super) provider: super::common_utils::MessagesProvider,
+struct ProviderCall {
+    identity: ProviderIdentity,
+    wire: WireRequest,
+    provider: MessagesProvider,
     signer: Option<litellm_auth_aws::SigV4Signer>,
     timeout: Option<Duration>,
     stream: bool,
 }
 
 impl ProviderCall {
-    pub fn cacheable(&self) -> bool {
+    fn cacheable(&self) -> bool {
         self.signer.is_none()
     }
 }
 
-impl MessagesRoute {
-    pub(super) async fn prepare_outbound(
-        &self,
-        request: ProviderMessagesRequest,
-        context: &CallContext<'_, impl Interceptors<Error>>,
-    ) -> Result<ProviderCall, Error> {
-        let ProviderMessagesRequest {
-            provider,
-            url,
-            body,
-            environment,
-            timeout,
-            api_key,
-        } = request;
-        let request_context = RequestContext {
-            model: body.model.clone(),
-            custom_llm_provider: provider.as_str().to_string(),
-            optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
-            secret_fields: Vec::new(),
-            api_key,
-        };
-        let authenticated =
-            resolve_auth(&self.auth, environment, &|key| std::env::var(key).ok()).await?;
-        let identity = ProviderIdentity {
-            model: request_context.model.clone(),
-            provider: request_context.custom_llm_provider.clone(),
-        };
-        let wire = context
-            .interceptors
-            .before_provider_request(
-                WireRequest {
-                    url,
-                    headers: authenticated.headers,
-                    body: provider
-                        .config()
-                        .wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
-                },
-                request_context,
-            )
-            .await?;
-        let stream = match wire.body.get("stream") {
-            None | Some(Value::Null) => false,
-            Some(Value::Bool(stream)) => *stream,
-            Some(value) => {
-                return Err(Error::InvalidRequest(
-                    litellm_llms::ErrorDetail::InvalidValue {
-                        field: "stream",
-                        expected: "a boolean",
-                        actual: value.clone(),
-                    },
-                ));
-            }
-        };
-        Ok(ProviderCall {
-            identity,
-            wire,
-            provider,
-            signer: authenticated.signer,
-            timeout,
-            stream,
+pub(super) async fn execute(
+    route: &MessagesRoute,
+    provider: MessagesProvider,
+    model: String,
+    call: MessagesCall,
+    secrets: Secrets,
+    context: &CallContext<'_, impl Interceptors<Error>>,
+) -> Result<MessagesCallResponse, Error> {
+    let request = provider_call(
+        &route.auth,
+        provider,
+        model,
+        call,
+        secrets.as_ref(),
+        context,
+    )
+    .boxed()
+    .await?;
+    let cache = CallCache::<Messages>::from_wire(
+        route.cache.as_ref().filter(|_| request.cacheable()),
+        context.cache,
+        &request.identity,
+        &request.wire,
+    );
+    let identity = request.identity.clone();
+    let (output, source) = match cache.lookup().await {
+        Some(hit) => hit,
+        None => (
+            call_provider(&route.http, request, context).await?,
+            ResultSource::Provider,
+        ),
+    };
+    let reported = match &output {
+        MessagesCallResponse::Complete(response) => provider.config().reported_cost(response),
+        MessagesCallResponse::Stream { .. } => None,
+    };
+    context
+        .result_ready(ExecutionFacts {
+            provider: identity,
+            cost: Cost::settle(&source, reported),
+            source: source.clone(),
         })
-    }
+        .await?;
+    Ok(cache.finish(output, &source).await)
+}
 
-    pub(super) async fn call_provider(
-        &self,
-        request: ProviderCall,
-        context: &CallContext<'_, impl Interceptors<Error>>,
-    ) -> Result<MessagesCallResponse, Error> {
-        let ProviderCall {
-            identity,
-            wire,
-            provider,
-            signer,
-            timeout,
-            stream,
-        } = request;
-        let provider_name = provider.as_str();
-        log_request_body(provider_name, stream, &wire.body);
-        let response = send(
-            &self.http,
-            Authenticated {
-                headers: wire.headers,
-                signer,
+async fn provider_call(
+    auth: &AuthServices,
+    provider: MessagesProvider,
+    model: String,
+    call: MessagesCall,
+    secrets: &(dyn Lookup + Send + Sync),
+    context: &CallContext<'_, impl Interceptors<Error>>,
+) -> Result<ProviderCall, Error> {
+    let MessagesCall {
+        body,
+        api_key,
+        api_base,
+        litellm_params,
+        extra_headers,
+        provider_specific_header,
+        timeout,
+        shaping,
+        ..
+    } = call;
+    let connection_params = litellm_router_types::LitellmParams {
+        api_base: api_base.clone().or(litellm_params.api_base.clone()),
+        ..litellm_params
+    };
+    let config = provider.config();
+    let env_lookup = |key: &str| secrets.get(key);
+
+    let sanitized = config.shape_request(
+        MessagesRequest { model, ..body },
+        shaping.settings.reasoning_auto_summary,
+    )?;
+    let trimmed =
+        without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
+
+    let scoped =
+        get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
+    let forwarded = string_headers(Some(
+        extra_headers.into_iter().flatten().chain(scoped).collect(),
+    ))?;
+    let validated = config.validate_environment(
+        forwarded,
+        api_key.as_deref(),
+        &trimmed.model,
+        &trimmed.params,
+        &connection_params,
+        &env_lookup,
+    )?;
+
+    let body = config.transform_anthropic_messages_request(
+        trimmed,
+        &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
+    )?;
+    let environment = ValidatedEnvironment {
+        headers: config.request_headers(
+            with_default_headers(validated.headers, config.default_headers()),
+            &body,
+        ),
+        auth: validated.auth,
+    };
+
+    let url = config.get_complete_url(
+        api_base.as_deref(),
+        &body.model,
+        &connection_params,
+        body.params.stream == Some(true),
+        &env_lookup,
+    )?;
+
+    let request_context = RequestContext {
+        model: body.model.clone(),
+        custom_llm_provider: provider.as_str().to_string(),
+        optional_params: serde_json::to_value(&body.params).map_err(serialize_failure)?,
+        secret_fields: Vec::new(),
+        api_key: api_key.map(SecretValue::new),
+    };
+    let authenticated = resolve_auth(auth, environment, &|key| std::env::var(key).ok()).await?;
+    let identity = ProviderIdentity {
+        model: request_context.model.clone(),
+        provider: request_context.custom_llm_provider.clone(),
+    };
+    let wire = context
+        .interceptors
+        .before_provider_request(
+            WireRequest {
+                url,
+                headers: authenticated.headers,
+                body: config.wire_body(serde_json::to_value(&body).map_err(serialize_failure)?),
             },
-            &wire.url,
-            &wire.body,
-            timeout,
+            request_context,
         )
         .await?;
-        if !response.status().is_success() {
-            return Err(provider_error(response).await);
-        }
-        let config = provider.config();
-        if stream {
-            return Ok(streaming_response(
-                response,
-                config.stream_decoder(),
-                provider_name,
+    let stream = match wire.body.get("stream") {
+        None | Some(Value::Null) => false,
+        Some(Value::Bool(stream)) => *stream,
+        Some(value) => {
+            return Err(Error::InvalidRequest(
+                litellm_llms::ErrorDetail::InvalidValue {
+                    field: "stream",
+                    expected: "a boolean",
+                    actual: value.clone(),
+                },
             ));
         }
-        let text = response.text().await.map_err(network)?;
-        log_response_body(&text);
-        context.response_received(&text).await?;
-        decode_response(config, &identity.model, &text)
-            .map(|message| MessagesCallResponse::Complete(Box::new(message)))
+    };
+    Ok(ProviderCall {
+        identity,
+        wire,
+        provider,
+        signer: authenticated.signer,
+        timeout: timeout.or_else(|| shaping.settings.request_timeout()),
+        stream,
+    })
+}
+
+fn without_additional_drop_params(
+    request: MessagesRequest,
+    paths: &[String],
+) -> Result<MessagesRequest, Error> {
+    if paths.is_empty() {
+        return Ok(request);
     }
+    let params = serde_json::to_value(request.params).map_err(invalid_request)?;
+    let trimmed = paths
+        .iter()
+        .fold(params, |params, path| delete_nested_value(params, path));
+    Ok(MessagesRequest {
+        params: serde_json::from_value(trimmed).map_err(invalid_request)?,
+        ..request
+    })
+}
+
+async fn call_provider(
+    http: &litellm_http::Client,
+    request: ProviderCall,
+    context: &CallContext<'_, impl Interceptors<Error>>,
+) -> Result<MessagesCallResponse, Error> {
+    let ProviderCall {
+        identity,
+        wire,
+        provider,
+        signer,
+        timeout,
+        stream,
+    } = request;
+    let provider_name = provider.as_str();
+    log_request_body(provider_name, stream, &wire.body);
+    let response = send(
+        http,
+        Authenticated {
+            headers: wire.headers,
+            signer,
+        },
+        &wire.url,
+        &wire.body,
+        timeout,
+    )
+    .await?;
+    if !response.status().is_success() {
+        return Err(provider_error(response).await);
+    }
+    let config = provider.config();
+    if stream {
+        return Ok(streaming_response(
+            response,
+            config.stream_decoder(),
+            provider_name,
+        ));
+    }
+    let text = response.text().await.map_err(network)?;
+    log_response_body(&text);
+    context.response_received(&text).await?;
+    decode_response(config, &identity.model, &text)
+        .map(|message| MessagesCallResponse::Complete(Box::new(message)))
 }
 
 fn serialize_failure(err: serde_json::Error) -> Error {
@@ -271,10 +394,482 @@ fn log_chunk(provider: &str, stage: &str, data: &bytes::Bytes) {
 
 #[cfg(test)]
 mod tests {
-    use rstest::rstest;
+    use litellm_llms_types::headers::ProviderSpecificHeaders;
+    use rstest::{fixture, rstest};
+    use serde_json::{Map, Value, json};
     use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
 
     use super::*;
+    use crate::{MessagesSettings, MessagesShaping, common_utils::messages_provider};
+    use litellm_inference::provider::resolve_llm_provider;
+
+    #[fixture]
+    fn shaping() -> MessagesShaping {
+        MessagesShaping::default()
+    }
+
+    fn body(value: Value) -> MessagesRequest {
+        serde_json::from_value(value).unwrap()
+    }
+
+    async fn provider_call_for(call: MessagesCall) -> Result<ProviderCall, Error> {
+        provider_call_with_secrets(call, &|_: &str| None).await
+    }
+
+    async fn provider_call_with_secrets(
+        call: MessagesCall,
+        secrets: &(dyn Lookup + Send + Sync),
+    ) -> Result<ProviderCall, Error> {
+        let provider = resolve_llm_provider(
+            &call.body.model,
+            call.custom_llm_provider.as_deref(),
+            "messages",
+        )?;
+        let config = messages_provider(provider.provider, provider.model)
+            .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider.provider).to_string()))?;
+        let model = provider.model.to_owned();
+        let interceptors = ();
+        let context = CallContext::new(&interceptors, litellm_inference::CallOptions::default());
+        provider_call(
+            &AuthServices::default(),
+            config,
+            model,
+            call,
+            secrets,
+            &context,
+        )
+        .await
+    }
+
+    async fn wire_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
+        provider_call_for(MessagesCall {
+            body: body(fields),
+            api_key: Some("sk-test".into()),
+            api_base: Some("https://anthropic.test".into()),
+            custom_llm_provider: Some("anthropic".into()),
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        })
+        .await
+        .map(|call| call.wire.body)
+    }
+
+    #[rstest]
+    #[case::api_key(
+        &[("ANTHROPIC_API_KEY", "sk-secret")],
+        &[("x-api-key", "sk-secret")],
+        "https://api.anthropic.com/v1/messages"
+    )]
+    #[case::auth_token(
+        &[("ANTHROPIC_AUTH_TOKEN", "token")],
+        &[("authorization", "Bearer token")],
+        "https://api.anthropic.com/v1/messages"
+    )]
+    #[case::api_base(
+        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")],
+        &[("x-api-key", "sk-secret")],
+        "https://gateway.test/v1/messages"
+    )]
+    #[case::sdk_base_url(
+        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")],
+        &[("x-api-key", "sk-secret")],
+        "https://sdk.test/v1/messages"
+    )]
+    #[tokio::test]
+    async fn credentials_and_base_come_from_the_resolved_secrets(
+        shaping: MessagesShaping,
+        #[case] secrets: &[(&str, &str)],
+        #[case] expected_auth: &[(&str, &str)],
+        #[case] expected_url: &str,
+    ) {
+        let lookup = |name: &str| {
+            secrets
+                .iter()
+                .find(|(key, _)| *key == name)
+                .map(|(_, value)| value.to_string())
+        };
+        let call = provider_call_with_secrets(
+            MessagesCall {
+                body: body(
+                    json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
+                ),
+                api_key: None,
+                api_base: None,
+                custom_llm_provider: Some("anthropic".into()),
+                litellm_params: Default::default(),
+                extra_headers: None,
+                provider_specific_header: None,
+                timeout: None,
+                shaping,
+            },
+            &lookup,
+        )
+        .await
+        .unwrap();
+        let auth: Vec<(&str, &str)> = call
+            .wire
+            .headers
+            .iter()
+            .filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization"))
+            .map(|(name, value)| (name.as_str(), value.as_str()))
+            .collect();
+        assert_eq!(
+            (auth.as_slice(), call.wire.url.as_str()),
+            (expected_auth, expected_url)
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn missing_credentials_are_reported_before_the_request_is_transformed(
+        shaping: MessagesShaping,
+    ) {
+        let result = provider_call_for(MessagesCall {
+            body: body(
+                json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}]}),
+            ),
+            api_key: None,
+            api_base: None,
+            custom_llm_provider: Some("anthropic".into()),
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        })
+        .await;
+        assert!(
+            matches!(
+                result,
+                Err(Error::Auth(litellm_auth::Error::MissingApiKey { .. }))
+            ),
+            "{:?}",
+            result.err()
+        );
+    }
+
+    #[rstest]
+    #[case::nothing_configured(None, None, None)]
+    #[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
+    #[case::call_timeout_wins(
+        Some(Duration::from_secs(30)),
+        Some(1800.0),
+        Some(Duration::from_secs(30))
+    )]
+    #[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
+    #[case::unusable_setting_ignored(None, Some(-1.0), None)]
+    #[case::zero_setting_ignored(None, Some(0.0), None)]
+    #[tokio::test]
+    async fn timeout_falls_back_to_the_configured_request_timeout(
+        shaping: MessagesShaping,
+        #[case] call_timeout: Option<Duration>,
+        #[case] request_timeout: Option<f64>,
+        #[case] expected: Option<Duration>,
+    ) {
+        let call = provider_call_for(MessagesCall {
+            body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
+            api_key: Some("sk-test".into()),
+            api_base: Some("https://anthropic.test".into()),
+            custom_llm_provider: Some("anthropic".into()),
+            litellm_params: Default::default(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: call_timeout,
+            shaping: MessagesShaping {
+                settings: MessagesSettings { request_timeout, ..shaping.settings },
+                ..shaping
+            },
+        })
+        .await
+        .unwrap();
+        assert_eq!(call.timeout, expected);
+    }
+
+    #[rstest]
+    #[case::top_level_and_nested_paths(
+        json!({
+            "max_tokens": 1024,
+            "thinking": {"type": "enabled", "budget_tokens": 2048},
+            "context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
+            "metadata": {"user_id": "u1"},
+            "tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}]
+        }),
+        &["thinking", "context_management", "tools[*].input_examples"],
+        json!({
+            "max_tokens": 1024,
+            "metadata": {"user_id": "u1"},
+            "tools": [{"name": "lookup", "input_schema": {"type": "object"}}]
+        }),
+    )]
+    #[case::no_paths(
+        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
+        &[],
+        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
+    )]
+    #[case::model_and_messages_are_never_dropped(
+        json!({"max_tokens": 16}),
+        &["model", "messages", "messages[0].content"],
+        json!({"max_tokens": 16}),
+    )]
+    #[tokio::test]
+    async fn wire_body_drops_configured_paths(
+        shaping: MessagesShaping,
+        #[case] fields: Value,
+        #[case] additional_drop_params: &[&str],
+        #[case] expected_fields: Value,
+    ) {
+        let with_messages = |fields: Value| -> Value {
+            let Value::Object(fields) = fields else {
+                unreachable!()
+            };
+            Value::Object(
+                [
+                    ("model".to_string(), json!("claude-test")),
+                    (
+                        "messages".to_string(),
+                        json!([{"role": "user", "content": "hi"}]),
+                    ),
+                ]
+                .into_iter()
+                .chain(fields)
+                .collect(),
+            )
+        };
+        let shaping = MessagesShaping {
+            settings: MessagesSettings {
+                additional_drop_params: additional_drop_params
+                    .iter()
+                    .map(ToString::to_string)
+                    .collect(),
+                ..shaping.settings
+            },
+            ..shaping
+        };
+        assert_eq!(
+            wire_body(with_messages(fields), shaping).await,
+            Ok(with_messages(expected_fields))
+        );
+    }
+
+    #[rstest]
+    #[case::model_prefix_picks_the_provider(
+        "azure_ai/claude-test",
+        None,
+        &[("x-priority", "extra"), ("x-scoped", "azure_ai")]
+    )]
+    #[case::explicit_provider(
+        "claude-test",
+        Some("anthropic"),
+        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
+    )]
+    #[case::provider_prefix_on_an_anthropic_model(
+        "anthropic/claude-test",
+        None,
+        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
+    )]
+    #[tokio::test]
+    async fn provider_specific_headers_follow_the_resolved_provider(
+        shaping: MessagesShaping,
+        #[case] model: &str,
+        #[case] custom_llm_provider: Option<&str>,
+        #[case] expected: &[(&str, &str)],
+    ) {
+        let configured: ProviderSpecificHeaders = serde_json::from_value(json!([
+            {"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}},
+            {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}}
+        ]))
+        .unwrap();
+        let call = provider_call_for(MessagesCall {
+            body: body(
+                json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
+            ),
+            api_key: Some("sk-test".into()),
+            api_base: Some("https://resource.services.ai.azure.com".into()),
+            custom_llm_provider: custom_llm_provider.map(Into::into),
+            litellm_params: Default::default(),
+            extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
+            provider_specific_header: Some(configured),
+            timeout: None,
+            shaping,
+        })
+        .await
+        .unwrap();
+        let caller_headers: Vec<(&str, &str)> = call
+            .wire
+            .headers
+            .iter()
+            .filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped"))
+            .map(|(name, value)| (name.as_str(), value.as_str()))
+            .collect();
+        assert_eq!(caller_headers, expected);
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn wire_body_carries_the_provider_stripped_model(shaping: MessagesShaping) {
+        assert_eq!(
+            wire_body(
+                json!({
+                    "model": "anthropic/claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 16
+                }),
+                shaping,
+            )
+            .await,
+            Ok(json!({
+                "model": "claude-test",
+                "messages": [{"role": "user", "content": "hi"}],
+                "max_tokens": 16
+            }))
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
+        let shaping = MessagesShaping {
+            settings: MessagesSettings {
+                reasoning_auto_summary: true,
+                additional_drop_params: vec!["thinking.display".to_string()],
+                ..shaping.settings
+            },
+            ..shaping
+        };
+        assert_eq!(
+            wire_body(
+                json!({
+                    "model": "claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 4096,
+                    "thinking": {"type": "enabled", "budget_tokens": 2048}
+                }),
+                shaping,
+            )
+            .await,
+            Ok(json!({
+                "model": "claude-test",
+                "messages": [{"role": "user", "content": "hi"}],
+                "max_tokens": 4096,
+                "thinking": {"type": "enabled", "budget_tokens": 2048}
+            }))
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(
+        shaping: MessagesShaping,
+    ) {
+        let shaping = MessagesShaping {
+            settings: MessagesSettings {
+                additional_drop_params: vec!["metadata.user_id".to_string()],
+                ..shaping.settings
+            },
+            ..shaping
+        };
+        assert!(matches!(
+            wire_body(
+                json!({
+                    "model": "claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 16,
+                    "metadata": {"user_id": 123}
+                }),
+                shaping,
+            )
+            .await,
+            Err(Error::InvalidRequest(_))
+        ));
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn wire_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) {
+        assert_eq!(
+            wire_body(
+                json!({
+                    "model": "claude-test",
+                    "messages": [{"role": "user", "content": "hi"}],
+                    "max_tokens": 16,
+                    "metadata": {"user_id": 123}
+                }),
+                shaping,
+            )
+            .await,
+            Err(Error::InvalidRequest(
+                litellm_llms::ErrorDetail::InvalidValue {
+                    field: "metadata.user_id",
+                    expected: "a string",
+                    actual: json!(123),
+                }
+            ))
+        );
+    }
+
+    #[rstest]
+    #[case::mantle(
+        "bedrock_mantle/claude-test",
+        "https://bedrock-mantle.us-west-2.api.aws",
+        "/anthropic/v1/messages",
+        "bedrock"
+    )]
+    #[case::platform(
+        "bedrock/claude_platform/claude-test",
+        "https://aws-external-anthropic.us-west-2.api.aws",
+        "/v1/messages",
+        "aws-external-anthropic"
+    )]
+    #[tokio::test]
+    async fn explicit_endpoint_region_is_used_by_the_signer(
+        shaping: MessagesShaping,
+        #[case] model: &str,
+        #[case] base: &str,
+        #[case] path: &str,
+        #[case] service: &str,
+    ) {
+        let call = provider_call_with_secrets(MessagesCall {
+            body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
+            api_key: None,
+            api_base: Some(base.into()),
+            custom_llm_provider: None,
+            litellm_params: crate::litellm_params(Map::from_iter([
+                ("model".into(), json!(model)),
+                ("aws_access_key_id".into(), json!("AKIDEXAMPLE")),
+                ("aws_secret_access_key".into(), json!("secret")),
+            ]))
+            .unwrap(),
+            extra_headers: None,
+            provider_specific_header: None,
+            timeout: None,
+            shaping,
+        }, &|name: &str| match name {
+            "AWS_REGION_NAME" => Some("us-east-1".into()),
+            "ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
+            _ => None,
+        })
+        .await
+        .unwrap();
+        assert_eq!(call.wire.url, format!("{base}{path}"));
+        let signed = outbound_request(
+            Authenticated {
+                headers: call.wire.headers,
+                signer: call.signer,
+            },
+            call.wire.url,
+            &call.wire.body,
+            None,
+        )
+        .unwrap();
+        let authorization = signed.header("authorization").unwrap_or_default();
+        assert!(
+            authorization.contains(&format!("/us-west-2/{service}/aws4_request")),
+            "{authorization}"
+        );
+    }
 
     #[rstest]
     #[case::event(

litellm-rust/crates/inference-messages/src/lib.rs

@@ -1,18 +1,19 @@
 mod common_utils;
 mod constants;
 mod handler;
-mod prepare;
 pub mod route;
 mod types;
 
 use futures_util::FutureExt;
 use litellm_auth::AuthServices;
-use litellm_host::interceptors::{Cost, ExecutionFacts, Interceptors, ResultSource};
-
-use litellm_inference::{caching::CallCache, context::CallContext};
+use litellm_host::interceptors::Interceptors;
+use litellm_inference::context::CallContext;
+use litellm_inference::provider::resolve_llm_provider;
 use litellm_secrets::source::SecretSource;
 use std::sync::Arc;
 
+use crate::common_utils::messages_provider;
+
 pub use litellm_inference::RouteError as Error;
 pub use types::{
     MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,
@@ -74,39 +75,23 @@ impl MessagesRoute {
         context: CallContext<'_, impl Interceptors<Error>>,
     ) -> Result<MessagesCallResponse, Error> {
         litellm_inference::diagnostic::call(async {
-            let prepared = prepare::prepare(call, self.secrets.as_ref()).await?;
+            let provider = resolve_llm_provider(
+                &call.body.model,
+                call.custom_llm_provider.as_deref(),
+                "messages",
+            )?;
+            let config = messages_provider(provider.provider, provider.model).ok_or_else(|| {
+                Error::InvalidProvider(<&str>::from(provider.provider).to_string())
+            })?;
             litellm_inference::diagnostic::provider(
-                &prepared.body.model,
-                prepared.provider.as_str(),
+                provider.model,
+                <&'static str>::from(provider.provider),
             );
-            let request = self.prepare_outbound(prepared, &context).boxed().await?;
-            let cache = CallCache::<route::Messages>::from_wire(
-                self.cache.as_ref().filter(|_| request.cacheable()),
-                context.cache,
-                &request.identity,
-                &request.wire,
-            );
-            let identity = request.identity.clone();
-            let config = request.provider.config();
-            let (output, source) = match cache.lookup().await {
-                Some(hit) => hit,
-                None => (
-                    self.call_provider(request, &context).await?,
-                    ResultSource::Provider,
-                ),
-            };
-            let reported = match &output {
-                MessagesCallResponse::Complete(response) => config.reported_cost(response),
-                MessagesCallResponse::Stream { .. } => None,
-            };
-            context
-                .result_ready(ExecutionFacts {
-                    provider: identity,
-                    cost: Cost::settle(&source, reported),
-                    source: source.clone(),
-                })
-                .await?;
-            Ok(cache.finish(output, &source).await)
+            let model = provider.model.to_owned();
+            let secrets = self.secrets.resolve(config.config().secret_names()).await?;
+            handler::execute(self, config, model, call, secrets, &context)
+                .boxed()
+                .await
         })
         .await
     }

litellm-rust/crates/inference-messages/src/prepare.rs

@@ -1,554 +0,0 @@
-use std::time::Duration;
-
-use litellm_auth::SecretValue;
-use litellm_core_utils::{
-    dot_notation_indexing::delete_nested_value,
-    get_provider_specific_headers::get_provider_specific_headers, settings::Lookup,
-};
-use litellm_http::request::with_default_headers;
-use litellm_llms::base_llm::{
-    auth::ValidatedEnvironment, messages::context::MessagesTransformContext,
-};
-use litellm_llms_types::formats::messages::MessagesRequest;
-use litellm_secrets::source::SecretSource;
-
-use super::{
-    Error, MessagesCall,
-    common_utils::{MessagesProvider, messages_provider, string_headers},
-    types::invalid_request,
-};
-use litellm_inference::provider::resolve_llm_provider;
-
-struct ResolvedProvider {
-    model: String,
-    provider: MessagesProvider,
-}
-
-pub(super) struct ProviderMessagesRequest {
-    pub(super) provider: MessagesProvider,
-    pub(super) url: String,
-    pub(super) body: MessagesRequest,
-    pub(super) environment: ValidatedEnvironment,
-    pub(super) timeout: Option<Duration>,
-    /// The caller's own credential, reported to the host beside the wire request.
-    pub(super) api_key: Option<SecretValue>,
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub(super) async fn prepare(
-    call: MessagesCall,
-    secrets: &dyn SecretSource,
-) -> Result<ProviderMessagesRequest, Error> {
-    let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?;
-    let secrets = secrets
-        .resolve(resolved.provider.config().secret_names())
-        .await?;
-    prepare_provider_request(call, resolved, secrets.as_ref())
-}
-
-fn resolve_provider(
-    model: &str,
-    custom_llm_provider: Option<&str>,
-) -> Result<ResolvedProvider, Error> {
-    let resolved = resolve_llm_provider(model, custom_llm_provider, "messages")?;
-    let provider = messages_provider(resolved.provider, resolved.model)
-        .ok_or_else(|| Error::InvalidProvider(<&str>::from(resolved.provider).to_string()))?;
-    Ok(ResolvedProvider {
-        model: resolved.model.to_string(),
-        provider,
-    })
-}
-
-fn prepare_provider_request(
-    call: MessagesCall,
-    resolved: ResolvedProvider,
-    secrets: &dyn Lookup,
-) -> Result<ProviderMessagesRequest, Error> {
-    let ResolvedProvider { model, provider } = resolved;
-    let MessagesCall {
-        body,
-        api_key,
-        api_base,
-        litellm_params,
-        extra_headers,
-        provider_specific_header,
-        timeout,
-        shaping,
-        ..
-    } = call;
-    let connection_params = litellm_router_types::LitellmParams {
-        api_base: api_base.clone().or(litellm_params.api_base.clone()),
-        ..litellm_params
-    };
-    let config = provider.config();
-    let env_lookup = |key: &str| secrets.get(key);
-
-    let sanitized = config.shape_request(
-        MessagesRequest { model, ..body },
-        shaping.settings.reasoning_auto_summary,
-    )?;
-    let trimmed =
-        without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?;
-    let transformed = config.transform_anthropic_messages_request(
-        trimmed,
-        &MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params),
-    )?;
-
-    let scoped =
-        get_provider_specific_headers(provider_specific_header.as_ref(), provider.as_str());
-    let forwarded = string_headers(Some(
-        extra_headers.into_iter().flatten().chain(scoped).collect(),
-    ))?;
-    let forwarded = config.request_headers(
-        with_default_headers(forwarded, config.default_headers()),
-        &transformed,
-    );
-    let environment = config.validate_environment(
-        forwarded,
-        api_key.as_deref(),
-        &transformed.model,
-        &connection_params,
-        &env_lookup,
-    )?;
-
-    let url = config.get_complete_url(
-        api_base.as_deref(),
-        &transformed.model,
-        &connection_params,
-        transformed.params.stream == Some(true),
-        &env_lookup,
-    )?;
-
-    Ok(ProviderMessagesRequest {
-        provider,
-        url,
-        body: transformed,
-        environment,
-        timeout: timeout.or_else(|| shaping.settings.request_timeout()),
-        api_key: api_key.map(SecretValue::new),
-    })
-}
-
-fn without_additional_drop_params(
-    request: MessagesRequest,
-    paths: &[String],
-) -> Result<MessagesRequest, Error> {
-    if paths.is_empty() {
-        return Ok(request);
-    }
-    let params = serde_json::to_value(request.params).map_err(invalid_request)?;
-    let trimmed = paths
-        .iter()
-        .fold(params, |params, path| delete_nested_value(params, path));
-    Ok(MessagesRequest {
-        params: serde_json::from_value(trimmed).map_err(invalid_request)?,
-        ..request
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use litellm_llms::base_llm::auth::resolve_auth;
-    use litellm_llms_types::headers::ProviderSpecificHeaders;
-    use rstest::{fixture, rstest};
-    use serde_json::{Map, Value, json};
-
-    use super::*;
-    use crate::{MessagesSettings, MessagesShaping};
-
-    #[fixture]
-    fn shaping() -> MessagesShaping {
-        MessagesShaping::default()
-    }
-
-    fn body(value: Value) -> MessagesRequest {
-        serde_json::from_value(value).unwrap()
-    }
-
-    fn prepare(call: MessagesCall) -> Result<ProviderMessagesRequest, Error> {
-        prepare_with_secrets(call, &|_: &str| None)
-    }
-
-    fn prepare_with_secrets(
-        call: MessagesCall,
-        secrets: &dyn Lookup,
-    ) -> Result<ProviderMessagesRequest, Error> {
-        let resolved = resolve_provider(&call.body.model, call.custom_llm_provider.as_deref())?;
-        prepare_provider_request(call, resolved, secrets)
-    }
-
-    /// The headers as they go on the wire, credential applied.
-    fn wire_headers(prepared: &ProviderMessagesRequest) -> Vec<(String, String)> {
-        tokio::runtime::Builder::new_current_thread()
-            .build()
-            .unwrap()
-            .block_on(resolve_auth(
-                &litellm_auth::AuthServices::default(),
-                prepared.environment.clone(),
-                &|_| None,
-            ))
-            .unwrap()
-            .headers
-    }
-
-    #[rstest]
-    #[case::api_key(
-        &[("ANTHROPIC_API_KEY", "sk-secret")],
-        &[("x-api-key", "sk-secret")],
-        "https://api.anthropic.com/v1/messages"
-    )]
-    #[case::auth_token(
-        &[("ANTHROPIC_AUTH_TOKEN", "token")],
-        &[("authorization", "Bearer token")],
-        "https://api.anthropic.com/v1/messages"
-    )]
-    #[case::api_base(
-        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")],
-        &[("x-api-key", "sk-secret")],
-        "https://gateway.test/v1/messages"
-    )]
-    #[case::sdk_base_url(
-        &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")],
-        &[("x-api-key", "sk-secret")],
-        "https://sdk.test/v1/messages"
-    )]
-    fn credentials_and_base_come_from_the_resolved_secrets(
-        shaping: MessagesShaping,
-        #[case] secrets: &[(&str, &str)],
-        #[case] expected_auth: &[(&str, &str)],
-        #[case] expected_url: &str,
-    ) {
-        let lookup = |name: &str| {
-            secrets
-                .iter()
-                .find(|(key, _)| *key == name)
-                .map(|(_, value)| value.to_string())
-        };
-        let prepared = prepare_with_secrets(
-            MessagesCall {
-                body: body(
-                    json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
-                ),
-                api_key: None,
-                api_base: None,
-                custom_llm_provider: Some("anthropic".into()),
-                litellm_params: Default::default(),
-                extra_headers: None,
-                provider_specific_header: None,
-                timeout: None,
-                shaping,
-            },
-            &lookup,
-        )
-        .unwrap();
-        let headers = wire_headers(&prepared);
-        let auth: Vec<(&str, &str)> = headers
-            .iter()
-            .filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization"))
-            .map(|(name, value)| (name.as_str(), value.as_str()))
-            .collect();
-        assert_eq!(
-            (auth.as_slice(), prepared.url.as_str()),
-            (expected_auth, expected_url)
-        );
-    }
-
-    #[rstest]
-    #[case::nothing_configured(None, None, None)]
-    #[case::configured_setting(None, Some(1800.0), Some(Duration::from_secs(1800)))]
-    #[case::call_timeout_wins(
-        Some(Duration::from_secs(30)),
-        Some(1800.0),
-        Some(Duration::from_secs(30))
-    )]
-    #[case::call_timeout_only(Some(Duration::from_secs(30)), None, Some(Duration::from_secs(30)))]
-    #[case::unusable_setting_ignored(None, Some(-1.0), None)]
-    #[case::zero_setting_ignored(None, Some(0.0), None)]
-    fn timeout_falls_back_to_the_configured_request_timeout(
-        shaping: MessagesShaping,
-        #[case] call_timeout: Option<Duration>,
-        #[case] request_timeout: Option<f64>,
-        #[case] expected: Option<Duration>,
-    ) {
-        let prepared = prepare(MessagesCall {
-            body: body(json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16})),
-            api_key: Some("sk-test".into()),
-            api_base: Some("https://anthropic.test".into()),
-            custom_llm_provider: Some("anthropic".into()),
-            litellm_params: Default::default(),
-            extra_headers: None,
-            provider_specific_header: None,
-            timeout: call_timeout,
-            shaping: MessagesShaping {
-                settings: MessagesSettings { request_timeout, ..shaping.settings },
-                ..shaping
-            },
-        })
-        .unwrap();
-        assert_eq!(prepared.timeout, expected);
-    }
-
-    fn prepared_body(fields: Value, shaping: MessagesShaping) -> Result<Value, Error> {
-        prepare(MessagesCall {
-            body: body(fields),
-            api_key: Some("sk-test".into()),
-            api_base: Some("https://anthropic.test".into()),
-            custom_llm_provider: Some("anthropic".into()),
-            litellm_params: Default::default(),
-            extra_headers: None,
-            provider_specific_header: None,
-            timeout: None,
-            shaping,
-        })
-        .map(|prepared| serde_json::to_value(prepared.body).unwrap())
-    }
-
-    #[rstest]
-    #[case::top_level_and_nested_paths(
-        json!({
-            "max_tokens": 1024,
-            "thinking": {"type": "enabled", "budget_tokens": 2048},
-            "context_management": {"edits": [{"type": "clear_thinking_20251015"}]},
-            "metadata": {"user_id": "u1"},
-            "tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}]
-        }),
-        &["thinking", "context_management", "tools[*].input_examples"],
-        json!({
-            "max_tokens": 1024,
-            "metadata": {"user_id": "u1"},
-            "tools": [{"name": "lookup", "input_schema": {"type": "object"}}]
-        }),
-    )]
-    #[case::no_paths(
-        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
-        &[],
-        json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}),
-    )]
-    #[case::model_and_messages_are_never_dropped(
-        json!({"max_tokens": 16}),
-        &["model", "messages", "messages[0].content"],
-        json!({"max_tokens": 16}),
-    )]
-    fn prepared_body_drops_configured_paths(
-        shaping: MessagesShaping,
-        #[case] fields: Value,
-        #[case] additional_drop_params: &[&str],
-        #[case] expected_fields: Value,
-    ) {
-        let with_messages = |fields: Value| -> Value {
-            let Value::Object(fields) = fields else {
-                unreachable!()
-            };
-            Value::Object(
-                [
-                    ("model".to_string(), json!("claude-test")),
-                    (
-                        "messages".to_string(),
-                        json!([{"role": "user", "content": "hi"}]),
-                    ),
-                ]
-                .into_iter()
-                .chain(fields)
-                .collect(),
-            )
-        };
-        let shaping = MessagesShaping {
-            settings: MessagesSettings {
-                additional_drop_params: additional_drop_params
-                    .iter()
-                    .map(ToString::to_string)
-                    .collect(),
-                ..shaping.settings
-            },
-            ..shaping
-        };
-        assert_eq!(
-            prepared_body(with_messages(fields), shaping),
-            Ok(with_messages(expected_fields))
-        );
-    }
-
-    #[rstest]
-    #[case::model_prefix_picks_the_provider(
-        "azure_ai/claude-test",
-        None,
-        &[("x-priority", "extra"), ("x-scoped", "azure_ai")]
-    )]
-    #[case::explicit_provider(
-        "claude-test",
-        Some("anthropic"),
-        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
-    )]
-    #[case::provider_prefix_on_an_anthropic_model(
-        "anthropic/claude-test",
-        None,
-        &[("x-priority", "scoped"), ("x-scoped", "anthropic")]
-    )]
-    fn provider_specific_headers_follow_the_resolved_provider(
-        shaping: MessagesShaping,
-        #[case] model: &str,
-        #[case] custom_llm_provider: Option<&str>,
-        #[case] expected: &[(&str, &str)],
-    ) {
-        let configured: ProviderSpecificHeaders = serde_json::from_value(json!([
-            {"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}},
-            {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}}
-        ]))
-        .unwrap();
-        let prepared = prepare(MessagesCall {
-            body: body(
-                json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}),
-            ),
-            api_key: Some("sk-test".into()),
-            api_base: Some("https://resource.services.ai.azure.com".into()),
-            custom_llm_provider: custom_llm_provider.map(Into::into),
-            litellm_params: Default::default(),
-            extra_headers: Some(Map::from_iter([("x-priority".into(), json!("extra"))])),
-            provider_specific_header: Some(configured),
-            timeout: None,
-            shaping,
-        })
-        .unwrap();
-        let caller_headers: Vec<(&str, &str)> = prepared
-            .environment
-            .headers
-            .iter()
-            .filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped"))
-            .map(|(name, value)| (name.as_str(), value.as_str()))
-            .collect();
-        assert_eq!(caller_headers, expected);
-    }
-
-    #[rstest]
-    fn prepared_body_carries_the_provider_stripped_model(shaping: MessagesShaping) {
-        assert_eq!(
-            prepared_body(
-                json!({
-                    "model": "anthropic/claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 16
-                }),
-                shaping,
-            ),
-            Ok(json!({
-                "model": "claude-test",
-                "messages": [{"role": "user", "content": "hi"}],
-                "max_tokens": 16
-            }))
-        );
-    }
-
-    #[rstest]
-    fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) {
-        let shaping = MessagesShaping {
-            settings: MessagesSettings {
-                reasoning_auto_summary: true,
-                additional_drop_params: vec!["thinking.display".to_string()],
-                ..shaping.settings
-            },
-            ..shaping
-        };
-        assert_eq!(
-            prepared_body(
-                json!({
-                    "model": "claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 4096,
-                    "thinking": {"type": "enabled", "budget_tokens": 2048}
-                }),
-                shaping,
-            ),
-            Ok(json!({
-                "model": "claude-test",
-                "messages": [{"role": "user", "content": "hi"}],
-                "max_tokens": 4096,
-                "thinking": {"type": "enabled", "budget_tokens": 2048}
-            }))
-        );
-    }
-
-    #[rstest]
-    fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(shaping: MessagesShaping) {
-        let shaping = MessagesShaping {
-            settings: MessagesSettings {
-                additional_drop_params: vec!["metadata.user_id".to_string()],
-                ..shaping.settings
-            },
-            ..shaping
-        };
-        assert!(matches!(
-            prepared_body(
-                json!({
-                    "model": "claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 16,
-                    "metadata": {"user_id": 123}
-                }),
-                shaping,
-            ),
-            Err(Error::InvalidRequest(_))
-        ));
-    }
-
-    #[rstest]
-    fn prepared_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) {
-        assert_eq!(
-            prepared_body(
-                json!({
-                    "model": "claude-test",
-                    "messages": [{"role": "user", "content": "hi"}],
-                    "max_tokens": 16,
-                    "metadata": {"user_id": 123}
-                }),
-                shaping,
-            ),
-            Err(Error::InvalidRequest(
-                litellm_llms::ErrorDetail::InvalidValue {
-                    field: "metadata.user_id",
-                    expected: "a string",
-                    actual: json!(123),
-                }
-            ))
-        );
-    }
-    #[rstest]
-    #[case::mantle(
-        "bedrock_mantle/claude-test",
-        "https://bedrock-mantle.us-west-2.api.aws",
-        "/anthropic/v1/messages",
-        "bedrock"
-    )]
-    #[case::platform(
-        "bedrock/claude_platform/claude-test",
-        "https://aws-external-anthropic.us-west-2.api.aws",
-        "/v1/messages",
-        "aws-external-anthropic"
-    )]
-    fn explicit_endpoint_region_is_used_by_the_signer(
-        shaping: MessagesShaping,
-        #[case] model: &str,
-        #[case] base: &str,
-        #[case] path: &str,
-        #[case] service: &str,
-    ) {
-        let prepared = prepare_with_secrets(MessagesCall {
-            body: body(json!({"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": "hi"}]})),
-            api_key: None,
-            api_base: Some(base.into()),
-            custom_llm_provider: None,
-            litellm_params: Default::default(),
-            extra_headers: None,
-            provider_specific_header: None,
-            timeout: None,
-            shaping,
-        }, &|name: &str| match name {
-            "AWS_REGION_NAME" => Some("us-east-1".into()),
-            "ANTHROPIC_AWS_WORKSPACE_ID" => Some("workspace".into()),
-            _ => None,
-        }).unwrap();
-        assert_eq!(prepared.url, format!("{base}{path}"));
-        assert!(
-            matches!(prepared.environment.auth, litellm_llms::base_llm::auth::AuthScheme::AwsSigV4 { region, service: actual, .. } if region == "us-west-2" && actual == service)
-        );
-    }
-}

litellm-rust/crates/inference-ocr/src/arguments.rs

@@ -44,8 +44,8 @@ pub fn consumed_optional_param_names(
     model: &str,
     custom_llm_provider: Option<&str>,
 ) -> Result<Vec<&'static str>, Error> {
-    let (model, config) = resolve_provider_config(model, custom_llm_provider)?;
-    let provider_fields = config.get_supported_ocr_params(&model);
+    let (provider, config) = resolve_provider_config(model, custom_llm_provider)?;
+    let provider_fields = config.get_supported_ocr_params(provider.model);
     let auth_fields: &[&str] = match config {
         OcrConfigKind::AwsTextract | OcrConfigKind::AwsTextractAnalyze => AWS_AUTH_OPTION_FIELDS,
         OcrConfigKind::AzureAi

litellm-rust/crates/inference-ocr/src/client.rs

@@ -1,81 +0,0 @@
-use litellm_host::observation::ObservationSender;
-use std::sync::Arc;
-
-use litellm_host::interceptors::Interceptors;
-use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient};
-use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
-
-use super::{
-    handler::perform_ocr_request,
-    types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest},
-};
-
-#[derive(Clone)]
-pub struct OcrRoute {
-    client: OcrClient,
-}
-
-impl OcrRoute {
-    pub fn new(client: OcrClient) -> Self {
-        Self { client }
-    }
-
-    pub async fn execute(
-        &self,
-        request: LiteLLMOcrRequest,
-        interceptors: &impl Interceptors<Error>,
-        observers: Option<ObservationSender>,
-    ) -> Result<LiteLLMOcrResponse, Error> {
-        litellm_host::lifecycle::observe_unary(
-            observers.clone(),
-            self.run(request, interceptors, observers.as_ref()),
-        )
-        .await
-    }
-
-    #[tracing::instrument(name = "litellm.route", skip_all, fields(
-        route = "ocr",
-        model = %request.model,
-        resolved_model = %request.model,
-        provider = <&str>::from(request.config.provider()),
-        stream = false,
-        outcome
-    ))]
-    pub(super) async fn run(
-        &self,
-        request: LiteLLMOcrRequest,
-        interceptors: &impl Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<LiteLLMOcrResponse, Error> {
-        litellm_inference::diagnostic::unary(async {
-            let caller_document = matches!(&request.document, OcrDocumentInput::Document(_));
-            let prepared = prepare_request_document(request).await?;
-            let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =
-                Box::pin(perform_ocr_request(
-                    &self.client,
-                    prepared,
-                    interceptors,
-                    caller_document,
-                    observers,
-                ));
-            execute.await
-        })
-        .await
-    }
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-async fn prepare_request_document(
-    request: LiteLLMOcrRequest<OcrDocumentInput>,
-) -> Result<ResolvedOcrRequest, Error> {
-    if let OcrDocumentInput::Document(_) = &request.document {
-        return request.map_document(super::document::prepare_document);
-    }
-    let logger = litellm_tracing::Logger::current();
-    let span = tracing::Span::current();
-    tokio::task::spawn_blocking(move || {
-        logger.scope(|| span.in_scope(|| request.map_document(super::document::prepare_document)))
-    })
-    .await
-    .map_err(|error| Error::DocumentTask(Arc::new(error)))?
-}

litellm-rust/crates/inference-ocr/src/document.rs

@@ -1,10 +1,10 @@
-use std::{collections::BTreeMap as Map, io::Read, path::Path};
+use std::{collections::BTreeMap as Map, io::Read, path::Path, sync::Arc};
 
 use base64::{Engine, engine::general_purpose::STANDARD};
 use litellm_llms::base_llm::ocr::{error::Error, transformation::OCR_INLINE_MAX_BYTES};
 use litellm_llms_types::formats::ocr::OcrDocument;
 
-use crate::types::OcrDocumentInput;
+use crate::types::{LiteLLMOcrRequest, OcrDocumentInput, ResolvedOcrRequest};
 
 pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
     match input {
@@ -24,6 +24,21 @@ pub fn prepare_document(input: OcrDocumentInput) -> Result<OcrDocument, Error> {
     }
 }
 
+pub(crate) async fn resolve_document(
+    request: LiteLLMOcrRequest<OcrDocumentInput>,
+) -> Result<ResolvedOcrRequest, Error> {
+    if let OcrDocumentInput::Document(_) = &request.document {
+        return request.map_document(prepare_document);
+    }
+    let logger = litellm_tracing::Logger::current();
+    let span = tracing::Span::current();
+    tokio::task::spawn_blocking(move || {
+        logger.scope(|| span.in_scope(|| request.map_document(prepare_document)))
+    })
+    .await
+    .map_err(|error| Error::DocumentTask(Arc::new(error)))?
+}
+
 pub fn read_path_document(path: &Path, mime_type: Option<&str>) -> Result<OcrDocument, Error> {
     let mut bytes = Vec::new();
     std::fs::File::open(path)

litellm-rust/crates/inference-ocr/src/handler.rs

@@ -4,29 +4,45 @@ use litellm_host::{lifecycle::ExecutionEvent, observation::ObservationSender};
 use litellm_llms::base_llm::ocr::{
     error::Error,
     handler::{CallHooks, OcrClient},
-    transformation::PreparedOcrRequest,
+    transformation::{OcrConnection, PreparedOcrRequest},
 };
 use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
+use litellm_secrets::source::Secrets;
 use serde_json::Value;
 
-use super::{arguments::is_secret_param, prepare::prepare_request, provider_config::OcrConfigKind};
-use crate::types::ResolvedOcrRequest;
+use super::{arguments::is_secret_param, provider_config::OcrConfigKind};
+use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
 
-pub(crate) async fn perform_ocr_request(
+pub(crate) async fn execute(
     client: &OcrClient,
-    request: ResolvedOcrRequest,
+    config: OcrConfigKind,
+    call: ResolvedOcrRequest,
+    secrets: Secrets,
     host: &impl Interceptors<Error>,
     caller_document: bool,
     observers: Option<&ObservationSender>,
 ) -> Result<LiteLLMOcrResponse, Error> {
-    request.response_format()?;
-    let config = request.config;
-    let secrets = client
-        .secret_source()
-        .resolve(&config.secret_names())
-        .await
-        .map_err(|error| Error::Secret(std::sync::Arc::new(error)))?;
-    let request = prepare_request(request, caller_document, client, secrets);
+    call.response_format()?;
+    let LiteLLMOcrRequest {
+        model,
+        document,
+        credentials,
+        transport,
+        optional_params,
+        input_sources,
+        azure_ad_token_provider,
+        ..
+    } = call;
+    let resolved = config.resolve_credentials(credentials, &secrets);
+    let request = PreparedOcrRequest {
+        model,
+        document,
+        connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
+        caller_document,
+        optional_params,
+        input_sources,
+        azure_ad_token_provider,
+    };
     let interceptors = OcrCallHooks::new(host, &request, config, observers);
     config.ocr(client, &request, &interceptors).await
 }
@@ -86,3 +102,257 @@ impl<H: Interceptors<Error>> CallHooks<Error> for OcrCallHooks<'_, H> {
         Box::pin(self.interceptors.after_provider_response(raw))
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use std::time::Duration;
+
+    use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
+    use litellm_llms_types::formats::ocr::OcrResponseFormat;
+    use rstest::rstest;
+    use serde_json::{Value, json};
+    use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
+
+    use super::*;
+    use crate::{
+        OcrRoute,
+        types::LiteLLMOcrRequest,
+        wire::{OcrWireRequest, decode_request},
+    };
+
+    fn client() -> OcrClient {
+        OcrClient::for_test(
+            litellm_http::Client::plain_for_test(),
+            litellm_http::Client::no_redirect_for_test(),
+        )
+        .with_secrets(litellm_inference_testing::no_secrets())
+    }
+
+    fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest {
+        decode_request(OcrWireRequest {
+            model: model.into(),
+            document,
+            api_key: Some(litellm_auth::SecretValue::new("test-key")),
+            api_base: Some(base.into()),
+            custom_llm_provider: None,
+            extra_headers: None,
+            optional_params: options.as_object().unwrap().clone(),
+            input_sources: Default::default(),
+            timeout_seconds: Some(2.0),
+        })
+        .unwrap()
+    }
+
+    async fn run_request(request: LiteLLMOcrRequest) -> Result<LiteLLMOcrResponse, Error> {
+        OcrRoute::new(client()).execute(request, &(), None).await
+    }
+
+    async fn upstream(request_count: u64, response: Value) -> MockServer {
+        let server = MockServer::start().await;
+        Mock::given(any())
+            .respond_with(ResponseTemplate::new(200).set_body_json(response))
+            .up_to_n_times(request_count)
+            .mount(&server)
+            .await;
+        server
+    }
+
+    fn image(url: &str) -> Value {
+        json!({"type": "image_url", "image_url": url})
+    }
+
+    fn body(request: &wiremock::Request) -> Value {
+        serde_json::from_slice(&request.body).unwrap()
+    }
+
+    fn assert_mistral_request(request: &wiremock::Request) {
+        assert_eq!(
+            request
+                .headers
+                .get("authorization")
+                .unwrap()
+                .to_str()
+                .unwrap(),
+            "Bearer test-key"
+        );
+        assert_eq!(
+            request
+                .headers
+                .get("content-type")
+                .unwrap()
+                .to_str()
+                .unwrap(),
+            "application/json"
+        );
+        assert_eq!(
+            body(request),
+            json!({
+                "model": "mistral-ocr-maas",
+                "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
+                "pages": [0, 2],
+                "include_image_base64": true,
+                "unknown": "preserved"
+            })
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() {
+        let upstream = upstream(
+            1,
+            json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}),
+        )
+        .await;
+        let request = request(
+            "cohere/parse",
+            &upstream.uri(),
+            image("https://example.com/original.png"),
+            json!({
+                "output_format": "markdown", "timeout": 30,
+                "extra_body": {
+                    "output_format": {"future": true},
+                    "document": {"type": "image_url", "image_url": "https://example.com/a.png",
+                        "provider_options": {"nested": [false, 0, null]}}
+                }
+            }),
+        );
+
+        run_request(request).await.unwrap();
+
+        let [request] = upstream
+            .received_requests()
+            .await
+            .unwrap()
+            .try_into()
+            .unwrap();
+        assert_eq!(
+            body(&request),
+            json!({
+                "model": "parse", "output_format": {"future": true},
+                "document": {"type": "image_url", "image_url": "https://example.com/a.png",
+                    "provider_options": {"nested": [false, 0, null]}}
+            })
+        );
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn explicit_null_options_use_defaults_before_http() {
+        let upstream = upstream(
+            1,
+            json!({"pages": [{"index": 0, "markdown": {"content": "hello"}}]}),
+        )
+        .await;
+        let request = request(
+            "cohere/parse",
+            &upstream.uri(),
+            image("https://example.com/a.png"),
+            json!({"output_format": null, "req_format": null}),
+        );
+        assert_eq!(
+            request.response_format().unwrap(),
+            OcrResponseFormat::Litellm
+        );
+
+        run_request(request).await.unwrap();
+
+        let [request] = upstream
+            .received_requests()
+            .await
+            .unwrap()
+            .try_into()
+            .unwrap();
+        let body = body(&request);
+        assert_eq!(body["output_format"], "markdown");
+        assert!(body.get("req_format").is_none());
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() {
+        let upstream = upstream(
+            2,
+            json!({
+                "pages": [{"index": 0, "markdown": "hello"}],
+                "extra": "preserved"
+            }),
+        )
+        .await;
+        let options = json!({
+            "pages": [0, 2],
+            "include_image_base64": true,
+            "vertex_project": "project-1",
+            "vertex_location": "us-central1",
+            "unknown": "preserved"
+        });
+        let document =
+            json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"});
+        let direct_call = request(
+            "mistral/mistral-ocr-maas",
+            &upstream.uri(),
+            document.clone(),
+            options.clone(),
+        );
+        let vertex_call = request(
+            "vertex_ai/mistral-ocr-maas",
+            &upstream.uri(),
+            document,
+            options,
+        );
+        assert_eq!(direct_call.transport.timeout, Some(Duration::from_secs(2)));
+        assert_eq!(vertex_call.transport.timeout, Some(Duration::from_secs(2)));
+        let direct_response = run_request(direct_call).await.unwrap().into_json();
+        let vertex_response = run_request(vertex_call).await.unwrap().into_json();
+
+        let [direct_http, vertex_http] = upstream
+            .received_requests()
+            .await
+            .unwrap()
+            .try_into()
+            .unwrap();
+        assert_eq!(direct_http.url.path(), "/v1/ocr");
+        assert_eq!(
+            vertex_http.url.path(),
+            "/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
+        );
+        assert_mistral_request(&direct_http);
+        assert_mistral_request(&vertex_http);
+        assert_eq!(direct_response, vertex_response);
+        assert_eq!(direct_response["model"], "mistral-ocr-maas");
+        assert_eq!(direct_response["object"], "ocr");
+        assert_eq!(direct_response["extra"], "preserved");
+    }
+
+    #[derive(serde::Deserialize)]
+    struct KnownParams {
+        pages: Option<Vec<i64>>,
+    }
+
+    #[rstest]
+    fn parsed_provider_params_separates_known_and_extra_params() {
+        let arguments: CallArguments = serde_json::from_value(json!({
+            "pages": [0, 2],
+            "future_ocr_option": true,
+            "extra_body": {"provider_option": "value"}
+        }))
+        .unwrap();
+        let known: KnownParams = parse_options(&arguments).unwrap();
+        assert_eq!(known.pages, Some(vec![0, 2]));
+        assert_eq!(arguments["future_ocr_option"], true);
+        assert_eq!(arguments["extra_body"], json!({"provider_option": "value"}));
+        assert_eq!(
+            arguments
+                .iter()
+                .filter(|(name, _)| name.as_str() != "pages")
+                .count(),
+            2
+        );
+        assert_eq!(
+            compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(),
+            json!({
+                "pages": [0, 2], "future_ocr_option": true, "provider_option": "value"
+            })
+        );
+    }
+}

litellm-rust/crates/inference-ocr/src/lib.rs

@@ -1,10 +1,82 @@
 pub mod arguments;
-mod client;
-pub use client::OcrRoute;
 pub mod document;
 pub(crate) mod handler;
-pub(crate) mod prepare;
 pub mod provider_config;
 pub mod route;
 pub mod types;
 pub mod wire;
+
+use std::sync::Arc;
+
+use litellm_host::interceptors::Interceptors;
+use litellm_host::observation::ObservationSender;
+use litellm_llms::base_llm::ocr::{error::Error, handler::OcrClient};
+use litellm_llms_types::formats::ocr::LiteLLMOcrResponse;
+
+use crate::{
+    handler::execute,
+    types::{LiteLLMOcrRequest, OcrDocumentInput},
+};
+
+#[derive(Clone)]
+pub struct OcrRoute {
+    client: OcrClient,
+}
+
+impl OcrRoute {
+    pub fn new(client: OcrClient) -> Self {
+        Self { client }
+    }
+
+    pub async fn execute(
+        &self,
+        request: LiteLLMOcrRequest,
+        interceptors: &impl Interceptors<Error>,
+        observers: Option<ObservationSender>,
+    ) -> Result<LiteLLMOcrResponse, Error> {
+        litellm_host::lifecycle::observe_unary(
+            observers.clone(),
+            self.run(request, interceptors, observers.as_ref()),
+        )
+        .await
+    }
+
+    #[tracing::instrument(name = "litellm.route", skip_all, fields(
+        route = "ocr",
+        model = %call.model,
+        resolved_model = %call.model,
+        provider = <&str>::from(call.config.provider()),
+        stream = false,
+        outcome
+    ))]
+    async fn run(
+        &self,
+        call: LiteLLMOcrRequest,
+        interceptors: &impl Interceptors<Error>,
+        observers: Option<&ObservationSender>,
+    ) -> Result<LiteLLMOcrResponse, Error> {
+        litellm_inference::diagnostic::unary(async {
+            let caller_document = matches!(&call.document, OcrDocumentInput::Document(_));
+            let call = document::resolve_document(call).await?;
+            let config = call.config;
+            let secrets = self
+                .client
+                .secret_source()
+                .resolve(&config.secret_names())
+                .await
+                .map_err(|error| Error::Secret(Arc::new(error)))?;
+            let execute: futures_util::future::BoxFuture<'_, Result<LiteLLMOcrResponse, Error>> =
+                Box::pin(execute(
+                    &self.client,
+                    config,
+                    call,
+                    secrets,
+                    interceptors,
+                    caller_document,
+                    observers,
+                ));
+            execute.await
+        })
+        .await
+    }
+}

litellm-rust/crates/inference-ocr/src/prepare.rs

@@ -1,309 +0,0 @@
-use litellm_auth::{InputSource, SecretValue, Sourced};
-use litellm_core_utils::get_llm_provider_logic::LlmProviders;
-use litellm_llms::base_llm::ocr::{
-    handler::OcrClient,
-    transformation::{OcrConnection, OcrCredentialInputs, PreparedOcrRequest},
-};
-use litellm_secrets::source::Secrets;
-
-use crate::types::{LiteLLMOcrRequest, ResolvedOcrRequest};
-
-pub(crate) fn prepare_request(
-    request: ResolvedOcrRequest,
-    caller_document: bool,
-    client: &OcrClient,
-    secrets: Secrets,
-) -> PreparedOcrRequest {
-    let credentials = request.credentials.clone();
-    let (preferred_api_key_env, api_base_env) = match request.config.provider() {
-        LlmProviders::Mistral => (
-            Some("MISTRAL_AZURE_API_KEY"),
-            Some("MISTRAL_AZURE_API_BASE"),
-        ),
-        LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
-        _ => (None, None),
-    };
-    let secret = |name: &str| secrets.truthy(name);
-    let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
-        credentials.api_key.clone().or_else(|| {
-            preferred_api_key_env
-                .into_iter()
-                .chain(request.config.get_api_key_env_var())
-                .find_map(secret)
-                .map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
-        })
-    });
-    let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
-        credentials.api_base.clone().or_else(|| {
-            api_base_env
-                .and_then(secret)
-                .map(|value| Sourced::new(value, InputSource::Environment))
-        })
-    });
-    let resolved = request
-        .config
-        .resolve_connection_params(OcrCredentialInputs {
-            dynamic_api_key,
-            dynamic_api_base,
-            ..credentials
-        });
-    let LiteLLMOcrRequest {
-        model,
-        document,
-        transport,
-        optional_params,
-        input_sources,
-        azure_ad_token_provider,
-        ..
-    } = request;
-    PreparedOcrRequest {
-        model,
-        document,
-        connection: OcrConnection::new(resolved, transport, client.settings().clone(), secrets),
-        caller_document,
-        optional_params,
-        input_sources,
-        azure_ad_token_provider,
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::time::Duration;
-
-    use futures_util::future::BoxFuture;
-    use litellm_core_utils::call_arguments::{CallArguments, compose_body, parse_options};
-    use litellm_host::interceptors::WireRequest;
-    use litellm_llms::{
-        base_llm::ocr::{
-            error::Error,
-            handler::{CallHooks, OcrClient},
-            transformation::BaseOcrConfig,
-        },
-        cohere::ocr::transformation::CohereParseConfig,
-        mistral::ocr::transformation::MistralOcrConfig,
-        vertex_ai::ocr::transformation::VertexAiOcrConfig,
-    };
-    use litellm_llms_types::formats::ocr::OcrResponseFormat;
-    use serde_json::{Value, json};
-
-    use super::*;
-    use crate::{
-        document::prepare_document,
-        types::LiteLLMOcrRequest,
-        wire::{OcrWireRequest, decode_request},
-    };
-
-    /// Stands in for a host with no interceptors registered.
-    struct NoHooks;
-
-    impl CallHooks<Error> for NoHooks {
-        fn before_provider_request(
-            &self,
-            wire: WireRequest,
-        ) -> BoxFuture<'_, Result<WireRequest, Error>> {
-            Box::pin(async move { Ok(wire) })
-        }
-
-        fn response_received<'a>(&'a self, _body: &'a [u8]) -> BoxFuture<'a, Result<(), Error>> {
-            Box::pin(async { Ok(()) })
-        }
-    }
-
-    fn client() -> OcrClient {
-        OcrClient::for_test(
-            litellm_http::Client::plain_for_test(),
-            litellm_http::Client::no_redirect_for_test(),
-        )
-    }
-
-    fn request(model: &str, base: &str, document: Value, options: Value) -> LiteLLMOcrRequest {
-        decode_request(OcrWireRequest {
-            model: model.into(),
-            document,
-            api_key: Some(litellm_auth::SecretValue::new("test-key")),
-            api_base: Some(base.into()),
-            custom_llm_provider: None,
-            extra_headers: None,
-            optional_params: options.as_object().unwrap().clone(),
-            input_sources: Default::default(),
-            timeout_seconds: Some(2.0),
-        })
-        .unwrap()
-    }
-
-    fn prepared(request: LiteLLMOcrRequest) -> PreparedOcrRequest {
-        prepare_request(
-            request.map_document(prepare_document).unwrap(),
-            true,
-            &client(),
-            std::sync::Arc::new(litellm_core_utils::settings::ProcessEnvironment),
-        )
-    }
-
-    fn image(url: &str) -> Value {
-        json!({"type": "image_url", "image_url": url})
-    }
-
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn cohere_body_keeps_native_document_fields_and_untyped_overrides() {
-        let request = request(
-            "cohere/parse",
-            "https://example.com",
-            image("https://example.com/original.png"),
-            json!({
-                "output_format": "markdown", "timeout": 30,
-                "extra_body": {
-                    "output_format": {"future": true},
-                    "document": {"type": "image_url", "image_url": "https://example.com/a.png",
-                        "provider_options": {"nested": [false, 0, null]}}
-                }
-            }),
-        );
-
-        let http = CohereParseConfig
-            .prepare_request(&prepared(request), &client(), &NoHooks)
-            .await
-            .unwrap();
-
-        let body: Value = serde_json::from_slice(http.body()).unwrap();
-        assert_eq!(
-            body,
-            json!({
-                "model": "parse", "output_format": {"future": true},
-                "document": {"type": "image_url", "image_url": "https://example.com/a.png",
-                    "provider_options": {"nested": [false, 0, null]}}
-            })
-        );
-    }
-
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn explicit_null_options_use_defaults_before_http() {
-        let request = request(
-            "cohere/parse",
-            "https://example.com",
-            image("https://example.com/a.png"),
-            json!({"output_format": null, "req_format": null}),
-        );
-        assert_eq!(
-            request.response_format().unwrap(),
-            OcrResponseFormat::Litellm
-        );
-
-        let http = CohereParseConfig
-            .prepare_request(&prepared(request), &client(), &NoHooks)
-            .await
-            .unwrap();
-
-        let body: Value = serde_json::from_slice(http.body()).unwrap();
-        assert_eq!(body["output_format"], "markdown");
-        assert!(body.get("req_format").is_none());
-    }
-
-    #[rstest::rstest]
-    #[tokio::test]
-    async fn direct_and_vertex_mistral_build_the_same_request_and_share_normalization() {
-        let options = json!({
-            "pages": [0, 2],
-            "include_image_base64": true,
-            "vertex_project": "project-1",
-            "vertex_location": "us-central1",
-            "unknown": "preserved"
-        });
-        let document =
-            json!({"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"});
-        let direct = prepared(request(
-            "mistral/mistral-ocr-maas",
-            "https://mistral.test",
-            document.clone(),
-            options.clone(),
-        ));
-        let vertex = prepared(request(
-            "vertex_ai/mistral-ocr-maas",
-            "https://vertex.test",
-            document,
-            options,
-        ));
-
-        let direct_http = MistralOcrConfig
-            .prepare_request(&direct, &client(), &NoHooks)
-            .await
-            .unwrap();
-        let vertex_http = VertexAiOcrConfig
-            .prepare_request(&vertex, &client(), &NoHooks)
-            .await
-            .unwrap();
-
-        assert_eq!(direct_http.url(), "https://mistral.test/v1/ocr");
-        assert_eq!(
-            vertex_http.url(),
-            "https://vertex.test/v1/projects/project-1/locations/us-central1/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
-        );
-        for http in [&direct_http, &vertex_http] {
-            assert_eq!(http.header("authorization").unwrap(), "Bearer test-key");
-            assert_eq!(http.header("content-type").unwrap(), "application/json");
-            assert_eq!(http.timeout(), Some(Duration::from_secs(2)));
-            let body: Value = serde_json::from_slice(http.body()).unwrap();
-            assert_eq!(
-                body,
-                json!({
-                    "model": "mistral-ocr-maas",
-                    "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
-                    "pages": [0, 2],
-                    "include_image_base64": true,
-                    "unknown": "preserved"
-                })
-            );
-        }
-        let payload = serde_json::to_vec(
-            &json!({"pages": [{"index": 0, "markdown": "hello"}], "extra": "preserved"}),
-        )
-        .unwrap();
-        let direct_response = MistralOcrConfig
-            .transform_ocr_response(&direct.model, &payload, OcrResponseFormat::Litellm)
-            .unwrap()
-            .into_json();
-        let vertex_response = VertexAiOcrConfig
-            .transform_ocr_response(&vertex.model, &payload, OcrResponseFormat::Litellm)
-            .unwrap()
-            .into_json();
-        assert_eq!(direct_response, vertex_response);
-        assert_eq!(direct_response["model"], "mistral-ocr-maas");
-        assert_eq!(direct_response["object"], "ocr");
-        assert_eq!(direct_response["extra"], "preserved");
-    }
-
-    #[derive(serde::Deserialize)]
-    struct KnownParams {
-        pages: Option<Vec<i64>>,
-    }
-
-    #[rstest::rstest]
-    fn parsed_provider_params_separates_known_and_extra_params() {
-        let arguments: CallArguments = serde_json::from_value(json!({
-            "pages": [0, 2],
-            "future_ocr_option": true,
-            "extra_body": {"provider_option": "value"}
-        }))
-        .unwrap();
-        let known: KnownParams = parse_options(&arguments).unwrap();
-        assert_eq!(known.pages, Some(vec![0, 2]));
-        assert_eq!(arguments["future_ocr_option"], true);
-        assert_eq!(arguments["extra_body"], json!({"provider_option": "value"}));
-        assert_eq!(
-            arguments
-                .iter()
-                .filter(|(name, _)| name.as_str() != "pages")
-                .count(),
-            2
-        );
-        assert_eq!(
-            compose_body(&arguments, &json!({"pages": known.pages}), &["pages"]).unwrap(),
-            json!({
-                "pages": [0, 2], "future_ocr_option": true, "provider_option": "value"
-            })
-        );
-    }
-}

litellm-rust/crates/inference-ocr/src/provider_config.rs

@@ -1,5 +1,6 @@
-use litellm_core_utils::get_llm_provider_logic::LlmProviders;
-use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider};
+use litellm_auth::{InputSource, SecretValue, Sourced};
+use litellm_core_utils::get_llm_provider_logic::{LlmProviders, get_custom_llm_provider};
+use litellm_inference::provider::{ResolvedProvider, resolve_llm_provider};
 use litellm_llms::{
     aws_textract::ocr::{
         analyze_transformation::TextractAnalyzeDocumentConfig, common_utils::TextractOperation,
@@ -25,6 +26,7 @@ use litellm_llms::{
     },
 };
 use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrResponseFormat};
+use litellm_secrets::source::Secrets;
 
 macro_rules! with_config {
     ($kind:expr, $config:ident => $body:expr) => {
@@ -115,7 +117,18 @@ impl OcrConfigKind {
     }
 
     pub(crate) fn secret_names(self) -> Vec<&'static str> {
-        with_config!(self, config => config.secret_names())
+        let names = with_config!(self, config => config.secret_names());
+        let fallback_names: &[&str] = match self.provider() {
+            LlmProviders::Mistral => &["MISTRAL_AZURE_API_KEY", "MISTRAL_AZURE_API_BASE"],
+            LlmProviders::AzureAi => &["AZURE_AI_API_BASE"],
+            _ => &[],
+        };
+        let additional_names = fallback_names
+            .iter()
+            .copied()
+            .filter(|name| !names.contains(name))
+            .collect::<Vec<_>>();
+        names.into_iter().chain(additional_names).collect()
     }
 
     pub(crate) fn get_health_check_document(self) -> OcrDocument {
@@ -129,6 +142,43 @@ impl OcrConfigKind {
         with_config!(self, config => config.resolve_connection_params(inputs))
     }
 
+    pub(crate) fn resolve_credentials(
+        self,
+        credentials: OcrCredentialInputs,
+        secrets: &Secrets,
+    ) -> ResolvedOcrCredentials {
+        let (preferred_api_key_env, api_base_env) = match self.provider() {
+            LlmProviders::Mistral => (
+                Some("MISTRAL_AZURE_API_KEY"),
+                Some("MISTRAL_AZURE_API_BASE"),
+            ),
+            LlmProviders::AzureAi => (None, Some("AZURE_AI_API_BASE")),
+            _ => (None, None),
+        };
+        let secret = |name: &str| secrets.truthy(name);
+        let dynamic_api_key = credentials.dynamic_api_key.or_else(|| {
+            credentials.api_key.clone().or_else(|| {
+                preferred_api_key_env
+                    .into_iter()
+                    .chain(self.get_api_key_env_var())
+                    .find_map(secret)
+                    .map(|value| Sourced::new(SecretValue::new(value), InputSource::Environment))
+            })
+        });
+        let dynamic_api_base = credentials.dynamic_api_base.or_else(|| {
+            credentials.api_base.clone().or_else(|| {
+                api_base_env
+                    .and_then(secret)
+                    .map(|value| Sourced::new(value, InputSource::Environment))
+            })
+        });
+        self.resolve_connection_params(OcrCredentialInputs {
+            dynamic_api_key,
+            dynamic_api_base,
+            ..credentials
+        })
+    }
+
     pub(crate) async fn ocr(
         self,
         client: &OcrClient,
@@ -164,7 +214,7 @@ pub fn passthrough_response(
     endpoint: &str,
     body: &[u8],
 ) -> Result<Option<LiteLLMOcrResponse>, Error> {
-    let (model, config) = resolve_provider_config(model, Some("azure_ai"))?;
+    let (provider, config) = resolve_provider_config(model, Some("azure_ai"))?;
     let segments: Vec<&str> = endpoint
         .split('/')
         .filter(|segment| !segment.is_empty())
@@ -173,7 +223,7 @@ pub fn passthrough_response(
         OcrConfigKind::AzureAi => segments == AZURE_AI_OCR_PATH,
         OcrConfigKind::AzureCohere => segments == AZURE_COHERE_PARSE_PATH,
         OcrConfigKind::AzureDocumentIntelligence => {
-            segments == AzureDocumentIntelligenceOcrConfig::analyze_path(&model)?
+            segments == AzureDocumentIntelligenceOcrConfig::analyze_path(provider.model)?
         }
         other => {
             let provider: &'static str = other.provider().into();
@@ -183,24 +233,28 @@ pub fn passthrough_response(
     if !is_ocr_endpoint {
         return Ok(None);
     }
-    with_config!(config, config => config.transform_ocr_response(&model, body, OcrResponseFormat::Litellm))
+    with_config!(config, config => config.transform_ocr_response(provider.model, body, OcrResponseFormat::Litellm))
         .map(Some)
 }
 
-pub(crate) fn resolve_provider_config(
-    model: &str,
-    custom_llm_provider: Option<&str>,
-) -> Result<(String, OcrConfigKind), Error> {
-    let provider =
-        get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider {
-            model,
-            custom_llm_provider: LlmProviders::Mistral.into(),
-        });
-    let llm_provider = provider
-        .custom_llm_provider
-        .parse::<LlmProviders>()
-        .map_err(|_| Error::InvalidProvider(provider.custom_llm_provider.to_string()))?;
-    let config = match llm_provider {
+pub(crate) fn resolve_provider_config<'a>(
+    model: &'a str,
+    custom_llm_provider: Option<&'a str>,
+) -> Result<(ResolvedProvider<'a>, OcrConfigKind), Error> {
+    let provider = match resolve_llm_provider(model, custom_llm_provider, "ocr") {
+        Ok(provider) => provider,
+        Err(_) if get_custom_llm_provider(model, custom_llm_provider).is_none() => {
+            ResolvedProvider {
+                model,
+                provider: LlmProviders::Mistral,
+            }
+        }
+        Err(litellm_inference::RouteError::InvalidProvider(provider)) => {
+            return Err(Error::InvalidProvider(provider));
+        }
+        Err(error) => return Err(Error::InvalidProvider(error.to_string())),
+    };
+    let config = match provider.provider {
         LlmProviders::AwsTextract => match TextractOperation::from_model(provider.model)? {
             TextractOperation::DetectDocumentText => OcrConfigKind::AwsTextract,
             TextractOperation::AnalyzeDocument => OcrConfigKind::AwsTextractAnalyze,
@@ -227,11 +281,11 @@ pub(crate) fn resolve_provider_config(
         LlmProviders::VertexAi => OcrConfigKind::VertexAi,
         _ => {
             return Err(Error::InvalidProvider(
-                provider.custom_llm_provider.to_string(),
+                <&'static str>::from(provider.provider).to_string(),
             ));
         }
     };
-    Ok((provider.model.to_string(), config))
+    Ok((provider, config))
 }
 
 fn is_document_intelligence_model(model: &str) -> bool {
@@ -243,7 +297,7 @@ fn is_document_intelligence_model(model: &str) -> bool {
 mod tests {
     use std::collections::HashSet;
 
-    use litellm_auth::{InputSource, Sourced};
+    use litellm_auth::Sourced;
     use litellm_llms::{
         base_llm::ocr::document::InlineDocument, cohere::ocr::transformation::validate_document,
     };
@@ -484,8 +538,8 @@ mod tests {
         #[case] expected_config: OcrConfigKind,
     ) {
         let expected_model = qualified_model.split_once('/').unwrap().1;
-        let (model, config) = resolve_provider_config(qualified_model, None).unwrap();
-        assert_eq!(model, expected_model);
+        let (provider, config) = resolve_provider_config(qualified_model, None).unwrap();
+        assert_eq!(provider.model, expected_model);
         assert_eq!(config, expected_config);
     }
 
@@ -528,7 +582,7 @@ mod tests {
             expected_config
         );
         assert_eq!(
-            resolve_provider_config(model, None).unwrap().0,
+            resolve_provider_config(model, None).unwrap().0.model,
             model.split_once('/').unwrap().1
         );
     }

litellm-rust/crates/inference-ocr/src/types.rs

@@ -75,6 +75,8 @@ impl OcrConnectionInputs {
 
 pub struct LiteLLMOcrRequest<D = OcrDocumentInput> {
     pub model: String,
+    pub(crate) requested_model: String,
+    pub(crate) custom_llm_provider: Option<String>,
     pub document: D,
     pub credentials: OcrCredentialInputs,
     pub transport: OcrTransportConfig,
@@ -91,7 +93,8 @@ impl LiteLLMOcrRequest {
         custom_llm_provider: Option<&str>,
         optional_params: CallArguments,
     ) -> Result<Self, Error> {
-        let (model, config) = resolve_provider_config(&model, custom_llm_provider)?;
+        let (provider, config) = resolve_provider_config(&model, custom_llm_provider)?;
+        let resolved_model = provider.model.to_owned();
         let default_transport = OcrTransportConfig::default();
         let max_response_bytes = optional_params
             .get("max_response_bytes")
@@ -116,7 +119,9 @@ impl LiteLLMOcrRequest {
             .collect();
 
         Ok(Self {
-            model,
+            model: resolved_model,
+            requested_model: model,
+            custom_llm_provider: custom_llm_provider.map(str::to_owned),
             document: document.into(),
             credentials: OcrCredentialInputs::default(),
             transport,
@@ -135,6 +140,8 @@ impl<D> LiteLLMOcrRequest<D> {
     ) -> Result<LiteLLMOcrRequest<T>, E> {
         Ok(LiteLLMOcrRequest {
             model: self.model,
+            requested_model: self.requested_model,
+            custom_llm_provider: self.custom_llm_provider,
             document: map(self.document)?,
             credentials: self.credentials,
             transport: self.transport,
@@ -148,6 +155,8 @@ impl<D> LiteLLMOcrRequest<D> {
     pub fn with_document<T>(self, document: T) -> LiteLLMOcrRequest<T> {
         LiteLLMOcrRequest {
             model: self.model,
+            requested_model: self.requested_model,
+            custom_llm_provider: self.custom_llm_provider,
             document,
             credentials: self.credentials,
             transport: self.transport,

litellm-rust/crates/inference-ocr/tests/ocr/lifecycle.rs

@@ -66,6 +66,19 @@ async fn interception_runs_once_and_observers_receive_ordered_success_events() {
     assert_eq!(received(&upstream).await.len(), 1);
 }
 
+#[rstest::rstest]
+#[case::mutated_model("model-b")]
+#[tokio::test]
+async fn provider_request_uses_the_request_model_after_it_is_mutated(#[case] model: &str) {
+    let upstream = upstream([pages_response()]).await;
+    let mut request = ocr_request("mistral/model-a", &upstream.uri(), json!({}));
+    request.model = model.into();
+
+    perform(request).await.unwrap();
+
+    assert_eq!(only_request(&upstream).await.json()["model"], json!(model));
+}
+
 #[rstest::rstest]
 #[tokio::test]
 async fn a_blocking_before_send_prevents_the_call_and_emits_one_failure() {

litellm-rust/crates/inference-responses/AGENTS.md

@@ -2,3 +2,4 @@
 - This crate owns Responses API call orchestration. Payloads belong in `litellm-llms-types`, transformations in `llms/src/base_llm/responses` and `llms/src/<provider>/responses`
 - The HTTP route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks, cached through `litellm_inference::caching::execute_streaming`
 - WebSocket sessions (`src/websocket.rs`) stay separate from the HTTP call driver because a connection can accept multiple requests while receiving events
+- `src/ids.rs` owns the LiteLLM-managed ids handed back to callers (`resp_`, `encitem_`, `litellm_enc:`, `cntr_`) and must stay fully compatible with `litellm/responses/utils.py`

litellm-rust/crates/inference-responses/Cargo.toml

@@ -6,6 +6,7 @@ license.workspace = true
 repository.workspace = true
 
 [dependencies]
+base64.workspace = true
 bytes.workspace = true
 futures-util.workspace = true
 litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }

litellm-rust/crates/inference-responses/src/handler.rs

@@ -3,39 +3,61 @@ use litellm_host::observation::ObservationSender;
 use std::time::Duration;
 
 use futures_util::StreamExt;
-use litellm_host::interceptors::{Interceptors, RawResponse, WireRequest};
-use litellm_llms::base_llm::auth::{Authenticated, resolve_auth};
+use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
+use litellm_llms::base_llm::auth::{AuthScheme, Authenticated, resolve_auth};
+use litellm_secrets::source::Secrets;
 
 use super::{
-    Error,
-    types::{ProviderResponsesRequest, ResponsesOutput, ResponsesStreamHead},
+    Error, ResponsesRoute,
+    types::{ResponsesCall, ResponsesOutput, ResponsesProvider, ResponsesStreamHead},
 };
 
 pub(super) async fn execute(
-    http: &litellm_http::Client,
-    auth: &litellm_auth::AuthServices,
-    request: ProviderResponsesRequest,
-    cache: Option<litellm_cache_response::ScopedCache>,
+    route: &ResponsesRoute,
+    ResponsesProvider { identity, config }: ResponsesProvider,
+    call: ResponsesCall,
+    secrets: Secrets,
     cache_options: Option<litellm_cache_response::CachePolicy>,
     interceptors: &impl Interceptors<Error>,
     observers: Option<&ObservationSender>,
 ) -> Result<ResponsesOutput, Error> {
-    let authenticated = resolve_auth(auth, request.environment, &|_| None).await?;
-    let identity = litellm_host::interceptors::ProviderIdentity {
-        model: request.context.model.clone(),
-        provider: request.context.custom_llm_provider.clone(),
+    let lookup = |name: &str| secrets.get(name);
+    let environment = config.validate_environment(
+        litellm_http::request::string_headers("responses", call.extra_headers)?,
+        call.api_key.as_deref(),
+        &lookup,
+    )?;
+    let url = config.get_complete_url(call.api_base.as_deref(), &lookup);
+    let context = RequestContext {
+        model: identity.model.clone(),
+        custom_llm_provider: identity.provider.clone(),
+        optional_params: serde_json::Value::Object(call.optional_params.clone()),
+        secret_fields: Vec::new(),
+        api_key: match &environment.auth {
+            AuthScheme::Credential { secret, .. } => Some(secret.clone()),
+            _ => None,
+        },
     };
+    let body = config.transform_responses_api_request(
+        &identity.model,
+        call.input,
+        call.optional_params,
+    )?;
+    let authenticated = resolve_auth(&route.auth, environment, &|_| None).await?;
     let wire = interceptors
         .before_provider_request(
             WireRequest {
-                url: request.url,
+                url,
                 headers: authenticated.headers,
-                body: request.body,
+                body,
             },
-            request.context,
+            context,
         )
         .await?;
-    let cache = cache.filter(|_| authenticated.signer.is_none());
+    let cache = route
+        .cache
+        .clone()
+        .filter(|_| authenticated.signer.is_none());
     let cache_request = litellm_inference::caching::CacheRequest::from_wire(
         identity,
         cache.as_ref().map(|_| &wire),
@@ -59,9 +81,9 @@ pub(super) async fn execute(
                 },
                 wire.url,
                 &wire.body,
-                Some(request.timeout.unwrap_or(Duration::from_secs(600))),
+                Some(call.timeout.unwrap_or(Duration::from_secs(600))),
             )?;
-            let response = litellm_inference::outbound::send(outbound, http)
+            let response = litellm_inference::outbound::send(outbound, &route.http)
                 .await
                 .map_err(network)?;
             let status = response.status().as_u16();
@@ -103,8 +125,7 @@ pub(super) async fn execute(
                 .map_err(Error::post_call)?;
             let value = serde_json::from_str(&body)
                 .map_err(|error| Error::InvalidResponse(error.to_string().into()))?;
-            request
-                .config
+            config
                 .transform_response_api_response(value)
                 .map(ResponsesOutput::Complete)
                 .map_err(Error::from)

litellm-rust/crates/inference-responses/src/ids.rs

@@ -0,0 +1,499 @@
+//! LiteLLM-managed identifiers on Responses payloads, fully compatible with `litellm/responses/utils.py`
+//!
+//! LiteLLM rewrites some ids in a Responses API result before handing it back, so that a later
+//! request carrying those ids can be routed to the provider and deployment that produced them,
+//! and then restored to what the provider originally issued. The native and Python paths serve
+//! the same traffic, so an id minted by either one must decode on the other.
+//!
+//! Each managed id is a fixed prefix followed by standard, padded base64 of a `;`-separated
+//! plaintext:
+//!
+//! | Id | Applied to | Decoded plaintext |
+//! |---|---|---|
+//! | `resp_<b64>` | the response `id` | `litellm:custom_llm_provider:{provider};model_id:{model_id};response_id:{id}` |
+//! | `encitem_<b64>` | the `id` of an output item with `encrypted_content` | `litellm:model_id:{model_id};item_id:{id}` |
+//! | `litellm_enc:<b64>;{content}` | that item's `encrypted_content` | `<b64>` is `model_id:{model_id}`, and the original content follows the `;` unencoded |
+//! | `cntr_<b64>` | `container_id` on output items, `code_interpreter_call`, and message annotations | `litellm:custom_llm_provider:{provider};model_id:{model_id};container_id:{id}` |
+//!
+//! For example, OpenAI's `resp_0a1b2c3d4e5f` served without a deployment id becomes
+//! `resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm`,
+//! which decodes to `litellm:custom_llm_provider:openai;model_id:None;response_id:resp_0a1b2c3d4e5f`.
+//!
+//! How they fit together:
+//! - `resp_` lets a follow-up's `previous_response_id` find the provider and deployment again.
+//!   [`encode_response`] stamps it on every response, unless the id is already managed.
+//! - `encitem_` and `litellm_enc:` carry encrypted-content affinity: encrypted reasoning only
+//!   decrypts under the API key that produced it, so the router pins replays to that deployment.
+//!   They are stamped only when the router sets `encrypted_content_affinity_enabled` and a
+//!   deployment `model_id` is known (see [`IdScope`]). Items without an `id`, such as Codex's,
+//!   carry the deployment in the wrapped content alone.
+//! - `cntr_` lets container calls find the provider and deployment that own a container. A
+//!   container id that already names a provider is left as it is.
+//! - On the way in, [`restore_input`] turns `encitem_` ids and `litellm_enc:` content in the
+//!   request input back into the provider's own bytes before the request is sent. The routing
+//!   decision itself stays in Python's `EncryptedContentAffinityCheck`.
+//!
+//! Python's quirks are kept on purpose:
+//! - A missing provider or `model_id` is written as `None` in `resp_` ids (an f-string of
+//!   `None`), but as an empty string in `cntr_` ids. [`decode_container_id`] reads both as absent.
+//! - Decoding follows Python's non-strict `base64.b64decode`: characters outside the alphabet
+//!   are skipped, trailing bits are allowed, and padding stripped in transit is added back.
+//! - A `resp_` id counts as managed when it decodes to at least three `;`-separated parts.
+
+use base64::{
+    Engine, alphabet,
+    engine::{GeneralPurpose, general_purpose::PAD},
+};
+use litellm_llms_types::formats::responses::ResponsesApiResponse;
+use serde_json::{Map, Value};
+
+const RESPONSE_PREFIX: &str = "resp_";
+const ITEM_PREFIX: &str = "encitem_";
+const CONTAINER_PREFIX: &str = "cntr_";
+const WRAPPED_CONTENT_PREFIX: &str = "litellm_enc:";
+
+/// Python's non-strict `base64.b64decode`: characters outside the alphabet are skipped and
+/// non-zero trailing bits are accepted
+const LENIENT: GeneralPurpose = GeneralPurpose::new(
+    &alphabet::STANDARD,
+    PAD.with_decode_allow_trailing_bits(true),
+);
+
+/// Where a response came from, as encoded into the ids LiteLLM hands back
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct IdScope {
+    pub provider: Option<String>,
+    pub model_id: Option<String>,
+    pub encrypted_content_affinity: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct DecodedContainerId {
+    pub provider: Option<String>,
+    pub model_id: Option<String>,
+    pub container_id: String,
+}
+
+/// Rewrites the response id, encrypted-content items and container ids the way the Python
+/// Responses path does before returning to the caller. An already managed id is left untouched
+pub fn encode_response(response: &mut ResponsesApiResponse, scope: &IdScope) {
+    if is_managed_response_id(&response.id) {
+        return;
+    }
+    response.id = response_id(scope, &response.id);
+    if let Some(model_id) = scope
+        .model_id
+        .as_deref()
+        .filter(|_| scope.encrypted_content_affinity)
+        .filter(|model_id| !model_id.is_empty())
+    {
+        response
+            .output
+            .iter_mut()
+            .filter_map(Value::as_object_mut)
+            .for_each(|item| encode_encrypted_item(item, model_id));
+    }
+    response
+        .output
+        .iter_mut()
+        .for_each(|item| encode_containers(item, scope));
+}
+
+/// Restores encrypted-content item ids and contents that LiteLLM encoded on an earlier response,
+/// so the provider receives what it originally issued
+pub fn restore_input(input: &mut Value) {
+    let Some(items) = input.as_array_mut() else {
+        return;
+    };
+    for item in items.iter_mut().filter_map(Value::as_object_mut) {
+        if let Some(Value::String(id)) = item.get_mut("id")
+            && let Some((_, original)) = decode_encrypted_item_id(id)
+        {
+            *id = original;
+        }
+        if let Some(Value::String(content)) = item.get_mut("encrypted_content")
+            && let Some(original) = unwrap_encrypted_content(content)
+        {
+            *content = original;
+        }
+    }
+}
+
+pub fn response_id(scope: &IdScope, id: &str) -> String {
+    let assembled = format!(
+        "litellm:custom_llm_provider:{};model_id:{};response_id:{id}",
+        python_str(scope.provider.as_deref()),
+        python_str(scope.model_id.as_deref()),
+    );
+    format!("{RESPONSE_PREFIX}{}", LENIENT.encode(assembled))
+}
+
+pub fn is_managed_response_id(id: &str) -> bool {
+    decode_text(&id.replace(RESPONSE_PREFIX, ""))
+        .is_some_and(|decoded| decoded.split(';').count() >= 3)
+}
+
+pub fn encrypted_item_id(model_id: &str, id: &str) -> String {
+    format!(
+        "{ITEM_PREFIX}{}",
+        LENIENT.encode(format!("litellm:model_id:{model_id};item_id:{id}"))
+    )
+}
+
+pub fn decode_encrypted_item_id(id: &str) -> Option<(String, String)> {
+    let decoded = decode_text(&padded(id.strip_prefix(ITEM_PREFIX)?))?;
+    let (model_id, item_id) = decoded.split_once(';')?;
+    Some((
+        model_id.replace("litellm:model_id:", ""),
+        item_id.replace("item_id:", ""),
+    ))
+}
+
+pub fn wrap_encrypted_content(content: &str, model_id: &str) -> String {
+    format!(
+        "{WRAPPED_CONTENT_PREFIX}{};{content}",
+        LENIENT.encode(format!("model_id:{model_id}"))
+    )
+}
+
+/// The original content of a wrapped `encrypted_content`, or `None` when it is not wrapped
+pub fn unwrap_encrypted_content(content: &str) -> Option<String> {
+    if !content.starts_with(WRAPPED_CONTENT_PREFIX) {
+        return None;
+    }
+    let (metadata, original) = content.split_once(';')?;
+    decode_text(&padded(&metadata.replace(WRAPPED_CONTENT_PREFIX, "")))?;
+    Some(original.to_owned())
+}
+
+pub fn container_id(scope: &IdScope, id: &str) -> String {
+    let assembled = format!(
+        "litellm:custom_llm_provider:{};model_id:{};container_id:{id}",
+        scope.provider.as_deref().unwrap_or_default(),
+        scope.model_id.as_deref().unwrap_or_default(),
+    );
+    format!("{CONTAINER_PREFIX}{}", LENIENT.encode(assembled))
+}
+
+pub fn decode_container_id(id: &str) -> DecodedContainerId {
+    let unmanaged = || DecodedContainerId {
+        provider: None,
+        model_id: None,
+        container_id: id.to_owned(),
+    };
+    if !id.starts_with(CONTAINER_PREFIX) {
+        return unmanaged();
+    }
+    let Some(decoded) = decode_text(&id.replace(CONTAINER_PREFIX, "")) else {
+        return unmanaged();
+    };
+    let parsed = decoded
+        .strip_prefix("litellm:custom_llm_provider:")
+        .and_then(|rest| rest.split_once(';'))
+        .and_then(|(provider, rest)| {
+            let (model_id, rest) = rest.strip_prefix("model_id:")?.split_once(';')?;
+            let container_id = rest.strip_prefix("container_id:")?;
+            (!container_id.is_empty() && !container_id.contains('\n')).then_some((
+                provider,
+                model_id,
+                container_id,
+            ))
+        });
+    let Some((provider, model_id, container_id)) = parsed else {
+        return unmanaged();
+    };
+    let present = |value: &str| (!matches!(value, "" | "None")).then(|| value.to_owned());
+    DecodedContainerId {
+        provider: present(provider),
+        model_id: present(model_id),
+        container_id: container_id.to_owned(),
+    }
+}
+
+fn encode_encrypted_item(item: &mut Map<String, Value>, model_id: &str) {
+    let Some(Value::String(content)) = item.get_mut("encrypted_content") else {
+        return;
+    };
+    if content.is_empty() {
+        return;
+    }
+    *content = wrap_encrypted_content(content, model_id);
+    if let Some(Value::String(id)) = item.get_mut("id")
+        && !id.is_empty()
+    {
+        *id = encrypted_item_id(model_id, id);
+    }
+}
+
+fn encode_containers(item: &mut Value, scope: &IdScope) {
+    let Some(item) = item.as_object_mut() else {
+        return;
+    };
+    encode_container_field(item, scope);
+    if let Some(nested) = item
+        .get_mut("code_interpreter_call")
+        .and_then(Value::as_object_mut)
+    {
+        encode_container_field(nested, scope);
+    }
+    if item.get("type").and_then(Value::as_str) != Some("message") {
+        return;
+    }
+    let Some(content) = item.get_mut("content").and_then(Value::as_array_mut) else {
+        return;
+    };
+    content
+        .iter_mut()
+        .filter_map(|part| part.get_mut("annotations").and_then(Value::as_array_mut))
+        .flatten()
+        .for_each(|annotation| encode_containers(annotation, scope));
+}
+
+fn encode_container_field(item: &mut Map<String, Value>, scope: &IdScope) {
+    if let Some(Value::String(id)) = item.get_mut("container_id")
+        && decode_container_id(id).provider.is_none()
+    {
+        *id = container_id(scope, id);
+    }
+}
+
+fn python_str(value: Option<&str>) -> &str {
+    value.unwrap_or("None")
+}
+
+fn padded(value: &str) -> String {
+    let missing = value.len() % 4;
+    let padding = if missing == 0 { 0 } else { 4 - missing };
+    format!("{value}{}", "=".repeat(padding))
+}
+
+fn decode_text(value: &str) -> Option<String> {
+    let alphabet: String = value
+        .chars()
+        .filter(|character| {
+            character.is_ascii_alphanumeric() || matches!(character, '+' | '/' | '=')
+        })
+        .collect();
+    String::from_utf8(LENIENT.decode(alphabet).ok()?).ok()
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::json;
+
+    use super::*;
+
+    fn scope(provider: Option<&str>, model_id: Option<&str>, affinity: bool) -> IdScope {
+        IdScope {
+            provider: provider.map(str::to_owned),
+            model_id: model_id.map(str::to_owned),
+            encrypted_content_affinity: affinity,
+        }
+    }
+
+    fn response(id: &str, output: Value) -> ResponsesApiResponse {
+        serde_json::from_value(json!({"id": id, "model": "gpt-6.1-sol", "output": output})).unwrap()
+    }
+
+    #[rstest]
+    #[case::openai_without_model_id(
+        Some("openai"),
+        None,
+        "resp_0a1b2c3d4e5f",
+        "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm"
+    )]
+    #[case::openrouter_with_model_id(
+        Some("openrouter"),
+        Some("deploy-1"),
+        "gen-123",
+        "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5yb3V0ZXI7bW9kZWxfaWQ6ZGVwbG95LTE7cmVzcG9uc2VfaWQ6Z2VuLTEyMw=="
+    )]
+    fn response_ids_match_python(
+        #[case] provider: Option<&str>,
+        #[case] model_id: Option<&str>,
+        #[case] id: &str,
+        #[case] expected: &str,
+    ) {
+        assert_eq!(response_id(&scope(provider, model_id, false), id), expected);
+    }
+
+    #[rstest]
+    #[case::raw_short_id("resp_0a1b2c3d4e5f", false)]
+    #[case::raw_long_id("resp_68a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3", false)]
+    #[case::two_segments("resp_YTti", false)]
+    #[case::three_segments("resp_YTtiO2M=", true)]
+    #[case::managed(
+        "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOng=",
+        true
+    )]
+    fn managed_response_ids_match_python(#[case] id: &str, #[case] expected: bool) {
+        assert_eq!(is_managed_response_id(id), expected);
+    }
+
+    #[test]
+    fn encrypted_items_match_python() {
+        assert_eq!(
+            encrypted_item_id("deploy-1", "rs_0a1b"),
+            "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI="
+        );
+        assert_eq!(
+            wrap_encrypted_content("gAAAA", "deploy-1"),
+            "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA"
+        );
+    }
+
+    #[rstest]
+    #[case::padded("encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", Some(("deploy-1", "rs_0a1b")))]
+    #[case::padding_stripped_in_transit("encitem_bGl0ZWxsbTptb2RlbF9pZDptO2l0ZW1faWQ6cnNfMQ", Some(("m", "rs_1")))]
+    #[case::not_base64("encitem_notbase64!", None)]
+    #[case::unmanaged("rs_0a1b", None)]
+    fn encrypted_item_ids_decode_like_python(
+        #[case] id: &str,
+        #[case] expected: Option<(&str, &str)>,
+    ) {
+        assert_eq!(
+            decode_encrypted_item_id(id),
+            expected.map(|(model_id, item_id)| (model_id.to_owned(), item_id.to_owned()))
+        );
+    }
+
+    #[rstest]
+    #[case::unpadded_metadata("litellm_enc:bW9kZWxfaWQ6bQ;gAAAA", Some("gAAAA"))]
+    #[case::no_separator("litellm_enc:nosemicolon", None)]
+    #[case::unwrapped("gAAAA", None)]
+    fn encrypted_content_unwraps_like_python(
+        #[case] content: &str,
+        #[case] expected: Option<&str>,
+    ) {
+        assert_eq!(unwrap_encrypted_content(content).as_deref(), expected);
+    }
+
+    #[rstest]
+    #[case::with_model_id(
+        Some("deploy-1"),
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpkZXBsb3ktMTtjb250YWluZXJfaWQ6Y250cl9hYmM="
+    )]
+    #[case::without_model_id(
+        None,
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfYWJj"
+    )]
+    fn container_ids_match_python(#[case] model_id: Option<&str>, #[case] expected: &str) {
+        assert_eq!(
+            container_id(&scope(Some("openai"), model_id, false), "cntr_abc"),
+            expected
+        );
+    }
+
+    #[rstest]
+    #[case::raw("cntr_abc", None, None, "cntr_abc")]
+    #[case::managed(
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDptO2NvbnRhaW5lcl9pZDpjbnRyXzE=",
+        Some("openai"),
+        Some("m"),
+        "cntr_1"
+    )]
+    #[case::empty_segments(
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOjttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfMQ==",
+        None,
+        None,
+        "cntr_1"
+    )]
+    #[case::python_none_provider(
+        "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOk5vbmU7bW9kZWxfaWQ6O2NvbnRhaW5lcl9pZDp4",
+        None,
+        None,
+        "x"
+    )]
+    fn container_ids_decode_like_python(
+        #[case] id: &str,
+        #[case] provider: Option<&str>,
+        #[case] model_id: Option<&str>,
+        #[case] container_id: &str,
+    ) {
+        assert_eq!(
+            decode_container_id(id),
+            DecodedContainerId {
+                provider: provider.map(str::to_owned),
+                model_id: model_id.map(str::to_owned),
+                container_id: container_id.to_owned(),
+            }
+        );
+    }
+
+    #[test]
+    fn encodes_response_items_under_affinity() {
+        let mut encoded = response(
+            "resp_1",
+            json!([
+                {"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []},
+                {"type": "reasoning", "encrypted_content": "gBBBB", "summary": []},
+                {"type": "code_interpreter_call", "id": "ci_1", "container_id": "cntr_abc"},
+                {"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [
+                    {"type": "container_file_citation", "container_id": "cntr_abc"}
+                ]}]}
+            ]),
+        );
+        let scope = scope(Some("openai"), Some("deploy-1"), true);
+        encode_response(&mut encoded, &scope);
+        let container = container_id(&scope, "cntr_abc");
+        assert_eq!(encoded.id, response_id(&scope, "resp_1"));
+        assert_eq!(
+            Value::Array(encoded.output),
+            json!([
+                {
+                    "type": "reasoning",
+                    "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=",
+                    "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA",
+                    "summary": []
+                },
+                {"type": "reasoning", "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gBBBB", "summary": []},
+                {"type": "code_interpreter_call", "id": "ci_1", "container_id": container},
+                {"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [
+                    {"type": "container_file_citation", "container_id": container}
+                ]}]}
+            ])
+        );
+    }
+
+    #[rstest]
+    #[case::affinity_disabled(scope(Some("openai"), Some("deploy-1"), false))]
+    #[case::no_model_id(scope(Some("openai"), None, true))]
+    fn leaves_encrypted_items_without_affinity(#[case] scope: IdScope) {
+        let output = json!([{"type": "reasoning", "id": "rs_1", "encrypted_content": "gAAAA", "summary": []}]);
+        let mut encoded = response("resp_1", output.clone());
+        encode_response(&mut encoded, &scope);
+        assert_eq!(Value::Array(encoded.output), output);
+    }
+
+    #[test]
+    fn leaves_managed_responses_untouched() {
+        let scope = scope(Some("openai"), Some("deploy-1"), true);
+        let id = response_id(&scope, "resp_1");
+        let output = json!([{"type": "code_interpreter_call", "container_id": "cntr_abc"}]);
+        let mut encoded = response(&id, output.clone());
+        encode_response(&mut encoded, &scope);
+        assert_eq!((encoded.id, Value::Array(encoded.output)), (id, output));
+    }
+
+    #[test]
+    fn restores_encrypted_items_in_input() {
+        let mut input = json!([
+            {
+                "type": "reasoning",
+                "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=",
+                "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA",
+                "summary": []
+            },
+            {"role": "user", "content": "hi"}
+        ]);
+        restore_input(&mut input);
+        assert_eq!(
+            input,
+            json!([
+                {"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []},
+                {"role": "user", "content": "hi"}
+            ])
+        );
+    }
+}

litellm-rust/crates/inference-responses/src/lib.rs

@@ -1,19 +1,36 @@
 use litellm_host::observation::ObservationSender;
 pub use litellm_inference::RouteError as Error;
 
+pub mod ids;
 pub mod route;
 pub mod types;
 pub mod websocket;
 
 mod handler;
-mod prepare;
 
 use std::sync::Arc;
 
 use litellm_auth::AuthServices;
+use litellm_core_utils::get_llm_provider_logic::{
+    CustomLlmProvider, LlmProviders, get_custom_llm_provider,
+};
 use litellm_host::interceptors::Interceptors;
+use litellm_host::interceptors::ProviderIdentity;
+use litellm_llms::{
+    base_llm::responses::transformation::BaseResponsesApiConfig,
+    openai::responses::transformation::OpenAiResponsesApiConfig,
+    openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
+};
 use litellm_secrets::source::SecretSource;
-use types::{ResponsesCall, ResponsesOutput};
+use types::{ResponsesCall, ResponsesOutput, ResponsesProvider};
+
+fn responses_provider(provider: LlmProviders) -> Option<&'static dyn BaseResponsesApiConfig> {
+    match provider {
+        LlmProviders::Openai => Some(&OpenAiResponsesApiConfig),
+        LlmProviders::Openrouter => Some(&OPENROUTER_RESPONSES_CONFIG),
+        _ => None,
+    }
+}
 
 #[derive(Clone)]
 pub struct ResponsesRoute {
@@ -77,34 +94,68 @@ impl ResponsesRoute {
         observers: Option<&ObservationSender>,
     ) -> Result<ResponsesOutput, Error> {
         litellm_inference::diagnostic::call(async {
-            self.run_provider(call, cache_options, interceptors, observers)
-                .await
+            let explicit = call
+                .custom_llm_provider
+                .as_deref()
+                .filter(|provider| !provider.is_empty());
+            let (provider_name, resolved_model) =
+                match get_custom_llm_provider(&call.model, explicit) {
+                    Some(CustomLlmProvider {
+                        model,
+                        custom_llm_provider,
+                    }) => (custom_llm_provider, model),
+                    None => ("openai", call.model.as_str()),
+                };
+            if resolved_model.is_empty() {
+                return Err(Error::InvalidProvider(call.model));
+            }
+            let provider = provider_name
+                .parse::<LlmProviders>()
+                .map_err(|_| Error::Unsupported("native HTTP responses provider"))?;
+            let config = responses_provider(provider)
+                .ok_or(Error::Unsupported("native HTTP responses provider"))?;
+            let wire_model = match provider {
+                LlmProviders::Openai if resolved_model.contains('/') => {
+                    return Err(Error::InvalidProvider(call.model));
+                }
+                LlmProviders::Openai => resolved_model.into(),
+                LlmProviders::Openrouter if explicit.is_some() && !resolved_model.contains('/') => {
+                    format!("openrouter/{resolved_model}")
+                }
+                LlmProviders::Openrouter => resolved_model.into(),
+                _ => return Err(Error::Unsupported("native HTTP responses provider")),
+            };
+            let provider = ResponsesProvider {
+                identity: ProviderIdentity {
+                    model: wire_model,
+                    provider: provider_name.into(),
+                },
+                config,
+            };
+            litellm_inference::diagnostic::provider(
+                &provider.identity.model,
+                &provider.identity.provider,
+            );
+            let secrets = self
+                .secrets
+                .resolve(
+                    provider
+                        .config
+                        .secret_names(call.api_key.as_deref(), call.api_base.as_deref()),
+                )
+                .await?;
+            let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
+                Box::pin(handler::execute(
+                    self,
+                    provider,
+                    call,
+                    secrets,
+                    cache_options,
+                    interceptors,
+                    observers,
+                ));
+            execute.await
         })
         .await
     }
-
-    async fn run_provider(
-        &self,
-        call: ResponsesCall,
-        cache_options: Option<litellm_cache_response::CachePolicy>,
-        interceptors: &impl Interceptors<Error>,
-        observers: Option<&ObservationSender>,
-    ) -> Result<ResponsesOutput, Error> {
-        let request = prepare::prepare(call, self.secrets.as_ref()).await?;
-        litellm_inference::diagnostic::provider(
-            &request.context.model,
-            &request.context.custom_llm_provider,
-        );
-        let execute: futures_util::future::BoxFuture<'_, Result<ResponsesOutput, Error>> =
-            Box::pin(handler::execute(
-                &self.http,
-                &self.auth,
-                request,
-                self.cache.clone(),
-                cache_options,
-                interceptors,
-                observers,
-            ));
-        execute.await
-    }
 }

litellm-rust/crates/inference-responses/src/prepare.rs

@@ -1,162 +0,0 @@
-use litellm_core_utils::get_llm_provider_logic::{
-    CustomLlmProvider, LlmProviders, get_custom_llm_provider,
-};
-use litellm_host::interceptors::{ProviderIdentity, RequestContext};
-use litellm_llms::{
-    base_llm::responses::transformation::BaseResponsesApiConfig,
-    openai::responses::transformation::OpenAiResponsesApiConfig,
-    openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
-};
-use litellm_secrets::source::SecretSource;
-use serde_json::Value;
-
-use super::{
-    Error,
-    types::{ProviderResponsesRequest, ResponsesCall},
-};
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub(super) async fn prepare(
-    call: ResponsesCall,
-    secrets: &dyn SecretSource,
-) -> Result<ProviderResponsesRequest, Error> {
-    let ResponsesProvider { identity, config } =
-        resolve_provider(&call.model, call.custom_llm_provider.as_deref())?;
-    let provider = identity.provider.as_str();
-    let model = identity.model.as_str();
-    let snapshot = secrets
-        .resolve(config.secret_names(call.api_key.as_deref(), call.api_base.as_deref()))
-        .await?;
-    let lookup = |name: &str| snapshot.get(name);
-    let environment = config.validate_environment(
-        litellm_http::request::string_headers("responses", call.extra_headers)?,
-        call.api_key.as_deref(),
-        &lookup,
-    )?;
-    let context = RequestContext {
-        model: model.into(),
-        custom_llm_provider: provider.into(),
-        optional_params: Value::Object(call.optional_params.clone()),
-        secret_fields: Vec::new(),
-        api_key: match &environment.auth {
-            litellm_llms::base_llm::auth::AuthScheme::Credential { secret, .. } => {
-                Some(secret.clone())
-            }
-            _ => None,
-        },
-    };
-    let body = config.transform_responses_api_request(model, call.input, call.optional_params)?;
-    Ok(ProviderResponsesRequest {
-        url: config.get_complete_url(call.api_base.as_deref(), &lookup),
-        config,
-        environment,
-        body,
-        context,
-        timeout: call.timeout,
-    })
-}
-
-pub(super) struct ResponsesProvider {
-    pub identity: ProviderIdentity,
-    pub config: &'static dyn BaseResponsesApiConfig,
-}
-
-/// Python's `get_llm_provider`: a bare model is OpenAI's, and with an explicit OpenRouter
-/// provider a model without a vendor segment keeps the `openrouter/` prefix as its wire id.
-pub(super) fn resolve_provider(
-    model: &str,
-    custom_llm_provider: Option<&str>,
-) -> Result<ResponsesProvider, Error> {
-    let explicit = custom_llm_provider.filter(|provider| !provider.is_empty());
-    let (provider, resolved) = match get_custom_llm_provider(model, explicit) {
-        Some(CustomLlmProvider {
-            model,
-            custom_llm_provider,
-        }) => (custom_llm_provider, model),
-        None => ("openai", model),
-    };
-    if resolved.is_empty() {
-        return Err(Error::InvalidProvider(model.into()));
-    }
-    let (config, wire_model): (&'static dyn BaseResponsesApiConfig, String) =
-        match provider.parse::<LlmProviders>() {
-            Ok(LlmProviders::Openai) if resolved.contains('/') => {
-                return Err(Error::InvalidProvider(model.into()));
-            }
-            Ok(LlmProviders::Openai) => (&OpenAiResponsesApiConfig, resolved.into()),
-            Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => (
-                &OPENROUTER_RESPONSES_CONFIG,
-                format!("openrouter/{resolved}"),
-            ),
-            Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()),
-            _ => return Err(Error::Unsupported("native HTTP responses provider")),
-        };
-    Ok(ResponsesProvider {
-        identity: ProviderIdentity {
-            model: wire_model,
-            provider: provider.into(),
-        },
-        config,
-    })
-}
-
-#[cfg(test)]
-mod tests {
-    use rstest::rstest;
-
-    use super::*;
-
-    /// Pinned against Python's `get_llm_provider` on 2026-10-11.
-    #[rstest]
-    #[case::bare("gpt-5", None, "openai", "gpt-5")]
-    #[case::openai_prefix("openai/gpt-5", None, "openai", "gpt-5")]
-    #[case::openai_explicit("gpt-5", Some("openai"), "openai", "gpt-5")]
-    #[case::openrouter_prefix(
-        "openrouter/anthropic/claude",
-        None,
-        "openrouter",
-        "anthropic/claude"
-    )]
-    #[case::openrouter_explicit(
-        "openrouter/anthropic/claude",
-        Some("openrouter"),
-        "openrouter",
-        "anthropic/claude"
-    )]
-    #[case::openrouter_explicit_unprefixed(
-        "anthropic/claude",
-        Some("openrouter"),
-        "openrouter",
-        "anthropic/claude"
-    )]
-    #[case::openrouter_native_id(
-        "openrouter/auto",
-        Some("openrouter"),
-        "openrouter",
-        "openrouter/auto"
-    )]
-    #[case::openrouter_native_id_implicit("openrouter/auto", None, "openrouter", "auto")]
-    fn providers_and_wire_models_match_python(
-        #[case] model: &str,
-        #[case] custom_llm_provider: Option<&str>,
-        #[case] provider: &str,
-        #[case] wire_model: &str,
-    ) {
-        let resolved = resolve_provider(model, custom_llm_provider).unwrap();
-        assert_eq!(resolved.identity.provider, provider);
-        assert_eq!(resolved.identity.model, wire_model);
-    }
-
-    #[rstest]
-    #[case::other_prefix("other/model", None)]
-    #[case::other_explicit("model", Some("other"))]
-    #[case::openai_with_vendor_segment("openai/other/model", None)]
-    #[case::empty_model("openai/", None)]
-    #[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
-    fn everything_else_is_rejected(#[case] model: &str, #[case] custom_llm_provider: Option<&str>) {
-        assert!(matches!(
-            resolve_provider(model, custom_llm_provider),
-            Err(Error::InvalidProvider(_) | Error::Unsupported(_))
-        ));
-    }
-}

litellm-rust/crates/inference-responses/src/types.rs

@@ -2,9 +2,8 @@ use std::time::Duration;
 
 use bytes::Bytes;
 use litellm_host::call::CallOutput;
-use litellm_llms::base_llm::{
-    auth::ValidatedEnvironment, responses::transformation::BaseResponsesApiConfig,
-};
+use litellm_host::interceptors::ProviderIdentity;
+use litellm_llms::base_llm::responses::transformation::BaseResponsesApiConfig;
 use litellm_llms_types::formats::responses::ResponsesApiResponse;
 use serde_json::{Map, Value};
 
@@ -21,17 +20,13 @@ pub struct ResponsesCall {
     pub timeout: Option<Duration>,
 }
 
+pub(crate) struct ResponsesProvider {
+    pub identity: ProviderIdentity,
+    pub config: &'static dyn BaseResponsesApiConfig,
+}
+
 pub struct ResponsesStreamHead {
     pub headers: Vec<(String, String)>,
 }
 
 pub type ResponsesOutput = CallOutput<ResponsesApiResponse, ResponsesStreamHead, Bytes, Error>;
-
-pub(super) struct ProviderResponsesRequest {
-    pub config: &'static dyn BaseResponsesApiConfig,
-    pub environment: ValidatedEnvironment,
-    pub url: String,
-    pub body: Value,
-    pub context: litellm_host::interceptors::RequestContext,
-    pub timeout: Option<Duration>,
-}

litellm-rust/crates/inference-responses/tests/responses.rs

@@ -7,6 +7,7 @@ use std::sync::Arc;
 use futures_util::TryStreamExt;
 use litellm_host::{call::HostedCompletion, lifecycle::CallEvent};
 use litellm_inference_responses::{
+    Error,
     route::Responses,
     types::{ResponsesCall, ResponsesOutput},
 };
@@ -32,6 +33,110 @@ fn call() -> ResponsesCall {
     }
 }
 
+#[rstest]
+#[case::bare_openai("gpt-5", None, "gpt-5")]
+#[case::openai_prefix("openai/gpt-5", None, "gpt-5")]
+#[case::empty_explicit_provider("openai/gpt-5", Some(""), "gpt-5")]
+#[case::explicit_openai("gpt-5", Some("openai"), "gpt-5")]
+#[case::openrouter_prefix("openrouter/anthropic/claude", None, "anthropic/claude")]
+#[case::explicit_openrouter_prefix(
+    "openrouter/anthropic/claude",
+    Some("openrouter"),
+    "anthropic/claude"
+)]
+#[case::explicit_openrouter_unprefixed("anthropic/claude", Some("openrouter"), "anthropic/claude")]
+#[case::explicit_openrouter_native_id("auto", Some("openrouter"), "openrouter/auto")]
+#[case::implicit_openrouter_native_id("openrouter/auto", None, "auto")]
+#[tokio::test]
+async fn provider_resolution_preserves_the_wire_model(
+    #[case] model: &str,
+    #[case] custom_llm_provider: Option<&str>,
+    #[case] expected_wire_model: &str,
+) {
+    let response = json!({
+        "id": "response-1",
+        "model": expected_wire_model,
+        "output": [{"type": "message", "content": []}],
+        "usage": {"total_tokens": 7}
+    });
+    let upstream = upstream([json_response(response)]).await;
+    responses_route(no_secrets())
+        .execute(
+            ResponsesCall {
+                model: model.into(),
+                api_key: Some("test-key".into()),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: custom_llm_provider.map(str::to_owned),
+                ..call()
+            },
+            &(),
+            None,
+        )
+        .await
+        .unwrap();
+
+    assert_eq!(
+        only_request(&upstream).await.json()["model"],
+        expected_wire_model
+    );
+}
+
+#[rstest]
+#[case::explicit_unknown_provider("test-model", Some("unknown-provider"))]
+#[case::unsupported_responses_provider("test-model", Some("anthropic"))]
+#[tokio::test]
+async fn unsupported_providers_fail_before_transport(
+    #[case] model: &str,
+    #[case] custom_llm_provider: Option<&str>,
+) {
+    let upstream = upstream([]).await;
+    let result = responses_route(no_secrets())
+        .execute(
+            ResponsesCall {
+                model: model.into(),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: custom_llm_provider.map(str::to_owned),
+                ..call()
+            },
+            &(),
+            None,
+        )
+        .await;
+
+    assert!(matches!(
+        result,
+        Err(Error::Unsupported("native HTTP responses provider"))
+    ));
+    assert!(received(&upstream).await.is_empty());
+}
+
+#[rstest]
+#[case::openai_with_vendor_segment("openai/other/model", None)]
+#[case::empty_model("openai/", None)]
+#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
+#[tokio::test]
+async fn invalid_provider_models_fail_before_transport(
+    #[case] model: &str,
+    #[case] custom_llm_provider: Option<&str>,
+) {
+    let upstream = upstream([]).await;
+    let result = responses_route(no_secrets())
+        .execute(
+            ResponsesCall {
+                model: model.into(),
+                api_base: Some(upstream.uri()),
+                custom_llm_provider: custom_llm_provider.map(str::to_owned),
+                ..call()
+            },
+            &(),
+            None,
+        )
+        .await;
+
+    assert!(matches!(result, Err(Error::InvalidProvider(_))));
+    assert!(received(&upstream).await.is_empty());
+}
+
 #[rstest]
 #[case::direct(false)]
 #[case::hosted(true)]

litellm-rust/crates/inference-transcription/src/handler.rs

@@ -1,28 +1,57 @@
 use std::time::Duration;
 
-use litellm_http::{Client, request::truncate_error_body};
-use litellm_llms::base_llm::auth::resolve_auth;
+use litellm_http::{
+    Client,
+    request::{string_headers, truncate_error_body, with_default_headers},
+};
+use litellm_llms::base_llm::{
+    audio_transcription::transformation::BaseAudioTranscriptionConfig,
+    auth::{ValidatedEnvironment, resolve_auth},
+};
+use litellm_secrets::source::Secrets;
 use serde_json::Value;
 
 use super::Error;
-use crate::{
-    constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::ProviderAudioTranscriptionRequest,
-};
+use crate::{constants::AUDIO_TRANSCRIPTION_TIMEOUT_SECS, types::AudioTranscriptionRequest};
+use litellm_inference::provider::ResolvedProvider;
 
-pub async fn execute_audio_transcription_provider_call(
+pub(super) async fn execute(
     http: &Client,
     auth: &litellm_auth::AuthServices,
-    request: ProviderAudioTranscriptionRequest,
+    config: &'static dyn BaseAudioTranscriptionConfig,
+    provider: ResolvedProvider<'_>,
+    call: AudioTranscriptionRequest<'_>,
+    secrets: Secrets,
 ) -> Result<Value, Error> {
-    let env_lookup = |key: &str| request.secrets.get(key);
-    let authenticated = resolve_auth(auth, request.environment.clone(), &env_lookup).await?;
+    let env_lookup = |key: &str| secrets.get(key);
+    let forwarded = string_headers("audio transcription", call.extra_headers)?;
+    let validated = config.validate_environment(
+        forwarded,
+        provider.model,
+        &call.optional_params,
+        &env_lookup,
+    )?;
+    let environment = ValidatedEnvironment {
+        headers: with_default_headers(validated.headers, config.default_headers()),
+        auth: validated.auth,
+    };
+    let url = config.get_complete_url(
+        call.api_base,
+        provider.model,
+        &call.optional_params,
+        &env_lookup,
+    )?;
+    let filtered_params = config.map_transcription_params(&call.optional_params);
+    let body = config
+        .transform_audio_transcription_request(provider.model, call.audio, filtered_params)?
+        .body;
+    let authenticated = resolve_auth(auth, environment, &env_lookup).await?;
     let outbound = litellm_inference::outbound::outbound_request(
         authenticated,
-        request.url.clone(),
-        &request.body,
+        url,
+        &body,
         Some(
-            request
-                .timeout
+            call.timeout
                 .unwrap_or(Duration::from_secs(AUDIO_TRANSCRIPTION_TIMEOUT_SECS)),
         ),
     )?;
@@ -47,8 +76,7 @@ pub async fn execute_audio_transcription_provider_call(
             error,
         ))
     })?;
-    Ok(request
-        .config
-        .transform_audio_transcription_response(&request.model, response_json)?
+    Ok(config
+        .transform_audio_transcription_response(provider.model, response_json)?
         .into_json())
 }

litellm-rust/crates/inference-transcription/src/lib.rs

@@ -2,16 +2,29 @@ pub mod types;
 pub use litellm_inference::RouteError as Error;
 mod constants;
 mod handler;
-mod prepare;
-pub use handler::execute_audio_transcription_provider_call;
+
 use litellm_auth::AuthServices;
+use litellm_core_utils::get_llm_provider_logic::LlmProviders;
+use litellm_inference::provider::resolve_llm_provider;
+use litellm_llms::{
+    base_llm::audio_transcription::transformation::BaseAudioTranscriptionConfig,
+    bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
+};
 use litellm_secrets::source::SecretSource;
-pub use prepare::prepare_audio_transcription_provider_call;
 use serde_json::Value;
 use std::sync::Arc;
 
 use crate::types::AudioTranscriptionRequest;
 
+fn audio_transcription_provider(
+    provider: LlmProviders,
+) -> Option<&'static dyn BaseAudioTranscriptionConfig> {
+    match provider {
+        LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
+        _ => None,
+    }
+}
+
 #[derive(Clone)]
 pub struct AudioTranscriptionRoute {
     http: litellm_http::Client,
@@ -32,6 +45,10 @@ impl AudioTranscriptionRoute {
         }
     }
 
+    pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
+        self.run(request).await
+    }
+
     #[tracing::instrument(name = "litellm.route", skip_all, fields(
         route = "audio_transcription",
         model = request.model,
@@ -40,13 +57,23 @@ impl AudioTranscriptionRoute {
         stream = false,
         outcome
     ))]
-    pub async fn execute(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
+    async fn run(&self, request: AudioTranscriptionRequest<'_>) -> Result<Value, Error> {
         litellm_inference::diagnostic::unary(async {
-            let request =
-                prepare_audio_transcription_provider_call(request, self.secrets.as_ref()).await?;
-            litellm_inference::diagnostic::provider(&request.model, &request.custom_llm_provider);
+            let provider = resolve_llm_provider(
+                request.model,
+                request.custom_llm_provider,
+                "audio transcription",
+            )?;
+            let config = audio_transcription_provider(provider.provider).ok_or_else(|| {
+                Error::InvalidProvider(<&str>::from(provider.provider).to_string())
+            })?;
+            litellm_inference::diagnostic::provider(
+                provider.model,
+                <&'static str>::from(provider.provider),
+            );
+            let secrets = self.secrets.resolve(&config.secret_names()).await?;
             let execute: futures_util::future::BoxFuture<'_, Result<Value, Error>> = Box::pin(
-                execute_audio_transcription_provider_call(&self.http, &self.auth, request),
+                handler::execute(&self.http, &self.auth, config, provider, request, secrets),
             );
             execute.await
         })

litellm-rust/crates/inference-transcription/src/prepare.rs

@@ -1,65 +0,0 @@
-use litellm_core_utils::get_llm_provider_logic::LlmProviders;
-use litellm_http::request::string_headers;
-use litellm_http::request::with_default_headers;
-use litellm_llms::{
-    base_llm::{
-        audio_transcription::transformation::BaseAudioTranscriptionConfig,
-        auth::ValidatedEnvironment,
-    },
-    bedrock::audio_transcription::BEDROCK_AUDIO_TRANSCRIPTION_CONFIG,
-};
-use litellm_secrets::source::SecretSource;
-
-use super::Error;
-use crate::types::{AudioTranscriptionRequest, ProviderAudioTranscriptionRequest};
-use litellm_inference::provider::resolve_llm_provider;
-
-fn provider_config(provider: LlmProviders) -> Option<&'static dyn BaseAudioTranscriptionConfig> {
-    match provider {
-        LlmProviders::Bedrock => Some(&BEDROCK_AUDIO_TRANSCRIPTION_CONFIG),
-        _ => None,
-    }
-}
-
-#[tracing::instrument(name = "litellm.prepare", level = "debug", skip_all)]
-pub async fn prepare_audio_transcription_provider_call(
-    request: AudioTranscriptionRequest<'_>,
-    secrets: &dyn SecretSource,
-) -> Result<ProviderAudioTranscriptionRequest, Error> {
-    let provider_info = resolve_llm_provider(
-        request.model,
-        request.custom_llm_provider,
-        "audio transcription",
-    )?;
-    let model = provider_info.model.to_string();
-    let config = provider_config(provider_info.provider)
-        .ok_or_else(|| Error::InvalidProvider(<&str>::from(provider_info.provider).to_string()))?;
-    let snapshot = secrets.resolve(&config.secret_names()).await?;
-    let env_lookup = |key: &str| snapshot.get(key);
-    let forwarded = string_headers("audio transcription", request.extra_headers)?;
-    let validated =
-        config.validate_environment(forwarded, &model, &request.optional_params, &env_lookup)?;
-    let environment = ValidatedEnvironment {
-        headers: with_default_headers(validated.headers, config.default_headers()),
-        auth: validated.auth,
-    };
-    let url = config.get_complete_url(
-        request.api_base,
-        &model,
-        &request.optional_params,
-        &env_lookup,
-    )?;
-    let filtered_params = config.map_transcription_params(&request.optional_params);
-    let transformed =
-        config.transform_audio_transcription_request(&model, request.audio, filtered_params)?;
-    Ok(ProviderAudioTranscriptionRequest {
-        model,
-        custom_llm_provider: <&str>::from(provider_info.provider).to_string(),
-        config,
-        url,
-        body: transformed.body,
-        environment,
-        secrets: snapshot,
-        timeout: request.timeout,
-    })
-}

litellm-rust/crates/inference-transcription/src/types.rs

@@ -1,9 +1,5 @@
-use litellm_secrets::source::Secrets;
 use std::time::Duration;
 
-use litellm_llms::base_llm::{
-    audio_transcription::transformation::BaseAudioTranscriptionConfig, auth::ValidatedEnvironment,
-};
 use serde_json::{Map, Value};
 
 pub struct AudioTranscriptionRequest<'a> {
@@ -16,37 +12,3 @@ pub struct AudioTranscriptionRequest<'a> {
     pub optional_params: Map<String, Value>,
     pub timeout: Option<Duration>,
 }
-
-#[derive(Clone)]
-pub struct ProviderAudioTranscriptionRequest {
-    pub model: String,
-    pub custom_llm_provider: String,
-    pub config: &'static dyn BaseAudioTranscriptionConfig,
-    pub url: String,
-    pub body: Value,
-    pub environment: ValidatedEnvironment,
-    pub secrets: Secrets,
-    pub timeout: Option<Duration>,
-}
-
-impl ProviderAudioTranscriptionRequest {
-    pub fn model(&self) -> &str {
-        &self.model
-    }
-
-    pub fn custom_llm_provider(&self) -> &str {
-        &self.custom_llm_provider
-    }
-
-    pub fn url(&self) -> &str {
-        &self.url
-    }
-
-    pub fn body(&self) -> &Value {
-        &self.body
-    }
-
-    pub fn with_body(self, body: Value) -> Self {
-        Self { body, ..self }
-    }
-}

litellm-rust/crates/llms/src/anthropic/messages/transformation.rs

@@ -74,6 +74,7 @@ impl BaseMessagesConfig for AnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -887,6 +888,7 @@ mod tests {
             headers(forwarded),
             api_key,
             "claude",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             &env(vars),
         )
@@ -1206,6 +1208,7 @@ mod tests {
             Vec::new(),
             None,
             "claude",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             &record,
         );

litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs

@@ -1,7 +1,7 @@
 use litellm_auth::SecretValue;
 use litellm_http::request::{has_bearer_auth, has_header};
 use litellm_llms_types::{
-    formats::messages::MessagesRequest,
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
     providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
 };
 use litellm_router_types::LitellmParams;
@@ -83,6 +83,7 @@ impl BaseMessagesConfig for AzureAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -237,6 +238,7 @@ mod tests {
                     .collect(),
                 api_key,
                 "claude",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &|_| None,
             )
@@ -622,6 +624,7 @@ mod tests {
             Vec::new(),
             None,
             "claude",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             &record,
         );

litellm-rust/crates/llms/src/base_llm/messages/transformation.rs

@@ -1,4 +1,6 @@
-use litellm_llms_types::formats::messages::{MessagesRequest, MessagesResponse};
+use litellm_llms_types::formats::messages::{
+    MessagesOptionalParams, MessagesRequest, MessagesResponse,
+};
 use litellm_router_types::LitellmParams;
 use serde_json::Value;
 
@@ -55,6 +57,7 @@ pub trait BaseMessagesConfig: Sync {
         headers: Headers,
         api_key: Option<&str>,
         model: &str,
+        optional_params: &MessagesOptionalParams,
         litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error>;
@@ -110,6 +113,7 @@ mod tests {
             headers: Headers,
             _api_key: Option<&str>,
             _model: &str,
+            _optional_params: &MessagesOptionalParams,
             _litellm_params: &LitellmParams,
             _env_lookup: &dyn Fn(&str) -> Option<String>,
         ) -> Result<ValidatedEnvironment, Error> {

litellm-rust/crates/llms/src/bedrock/claude_platform/messages/transformation.rs

@@ -5,7 +5,7 @@ use litellm_auth_aws::{AwsCredentialSource, resolve_aws_region};
 use litellm_core_utils::settings::resolve_non_empty;
 use litellm_http::request::{header_value, with_header, without_headers};
 use litellm_llms_types::{
-    formats::messages::MessagesRequest,
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
     providers::anthropic::{BetaProvider, DEFAULT_HEADERS, MESSAGES_PATH},
 };
 use litellm_router_types::LitellmParams;
@@ -60,6 +60,7 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        optional_params: &MessagesOptionalParams,
         params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -67,11 +68,8 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
             .filter(|value| !value.trim().is_empty())
             .or_else(|| {
                 WORKSPACE_FIELDS.iter().find_map(|field| {
-                    params
-                        .extra
-                        .get(*field)?
-                        .as_str()
-                        .filter(|value| !value.trim().is_empty())
+                    workspace_value(optional_params.extra.get(*field))
+                        .or_else(|| workspace_value(params.extra.get(*field)))
                 })
             });
         let workspace = resolve_non_empty(workspace, env, WORKSPACE_ENVS)
@@ -182,6 +180,10 @@ impl BaseMessagesConfig for BedrockClaudePlatformMessagesConfig {
     }
 }
 
+fn workspace_value(value: Option<&Value>) -> Option<&str> {
+    value?.as_str().filter(|value| !value.trim().is_empty())
+}
+
 fn platform_region(
     base: Option<&str>,
     params: &LitellmParams,
@@ -260,6 +262,7 @@ mod tests {
                     vec![],
                     Some("key"),
                     "claude-test",
+                    &MessagesOptionalParams::default(),
                     &LitellmParams::default(),
                     &|_| None
                 )
@@ -280,6 +283,7 @@ mod tests {
                 ],
                 Some("explicit"),
                 "claude-test",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &|name| (name == "ANTHROPIC_AWS_WORKSPACE_ID").then(|| "workspace".into()),
             )
@@ -345,7 +349,14 @@ mod tests {
             .get_complete_url(None, "claude-test", &params, false, &env)
             .unwrap();
         let environment = BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG
-            .validate_environment(vec![], None, "claude-test", &params, &env)
+            .validate_environment(
+                vec![],
+                None,
+                "claude-test",
+                &MessagesOptionalParams::default(),
+                &params,
+                &env,
+            )
             .unwrap();
         assert_eq!(
             url,

litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs

@@ -16,7 +16,7 @@ use litellm_auth_aws::{
     resolve_bedrock_region,
 };
 use litellm_llms_types::formats::messages::{
-    MessagesRequest,
+    MessagesOptionalParams, MessagesRequest,
     streaming::{MessagesStreamEvent, MessagesStreamUsage},
 };
 use litellm_llms_types::providers::bedrock::{
@@ -146,6 +146,7 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         model: &str,
+        _optional_params: &MessagesOptionalParams,
         litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -618,6 +619,7 @@ mod tests {
                 Vec::new(),
                 None,
                 "anthropic.claude-3",
+                &MessagesOptionalParams::default(),
                 &litellm_params,
                 &|_| None,
             )
@@ -658,6 +660,7 @@ mod tests {
                 vec![("authorization".into(), "Bearer forwarded".into())],
                 api_key,
                 "anthropic.claude-3",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &env,
             )

litellm-rust/crates/llms/src/bedrock_mantle/messages/transformation.rs

@@ -6,7 +6,7 @@ use litellm_auth_aws::bedrock_model_id_and_region;
 use litellm_core_utils::settings::resolve_non_empty;
 use litellm_http::request::{with_header, without_headers};
 use litellm_llms_types::{
-    formats::messages::MessagesRequest,
+    formats::messages::{MessagesOptionalParams, MessagesRequest},
     providers::anthropic::{AnthropicBeta, BetaProvider, BetaSet, DEFAULT_HEADERS},
 };
 use litellm_router_types::LitellmParams;
@@ -59,6 +59,7 @@ impl BaseMessagesConfig for BedrockMantleAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         model: &str,
+        _optional_params: &MessagesOptionalParams,
         params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -317,7 +318,14 @@ mod tests {
             "https://private.test/anthropic/v1/messages"
         );
         let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
-            .validate_environment(vec![], None, model, &params, &|_| None)
+            .validate_environment(
+                vec![],
+                None,
+                model,
+                &MessagesOptionalParams::default(),
+                &params,
+                &|_| None,
+            )
             .unwrap();
         assert!(
             matches!(environment.auth, AuthScheme::AwsSigV4 { region: actual, service: "bedrock", .. } if actual == region)
@@ -347,7 +355,14 @@ mod tests {
             .get_complete_url(None, "claude-test", &params, false, &|_| None)
             .unwrap();
         let environment = BEDROCK_MANTLE_MESSAGES_CONFIG
-            .validate_environment(vec![], None, "claude-test", &params, &|_| None)
+            .validate_environment(
+                vec![],
+                None,
+                "claude-test",
+                &MessagesOptionalParams::default(),
+                &params,
+                &|_| None,
+            )
             .unwrap();
         assert_eq!(
             url,
@@ -401,6 +416,7 @@ mod tests {
                 ],
                 key,
                 "claude-test",
+                &MessagesOptionalParams::default(),
                 &LitellmParams::default(),
                 &|name| match name {
                     "BEDROCK_MANTLE_API_KEY" => Some("mantle".into()),

litellm-rust/crates/llms/src/deepseek/messages/transformation.rs

@@ -104,6 +104,7 @@ impl BaseMessagesConfig for DeepSeekAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _litellm_params: &LitellmParams,
         env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -298,6 +299,7 @@ mod tests {
                 .collect(),
             api_key,
             "deepseek-v4-pro",
+            &MessagesOptionalParams::default(),
             &LitellmParams::default(),
             env,
         )

litellm-rust/crates/llms/src/edenai/messages/transformation.rs

@@ -36,6 +36,7 @@ impl BaseMessagesConfig for EdenAIAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {

litellm-rust/crates/llms/src/minimax/messages/transformation.rs

@@ -42,6 +42,7 @@ impl BaseMessagesConfig for MinimaxMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -183,7 +184,14 @@ mod tests {
                 .flatten()
         };
         let environment = MINIMAX_MESSAGES_CONFIG
-            .validate_environment(vec![], key, "test-model", &Default::default(), &env)
+            .validate_environment(
+                vec![],
+                key,
+                "test-model",
+                &MessagesOptionalParams::default(),
+                &Default::default(),
+                &env,
+            )
             .unwrap();
         assert!(matches!(
             environment.auth,

litellm-rust/crates/llms/src/openrouter/messages/transformation.rs

@@ -34,6 +34,7 @@ impl BaseMessagesConfig for OpenRouterAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {

litellm-rust/crates/llms/src/tencent/messages/transformation.rs

@@ -43,6 +43,7 @@ impl BaseMessagesConfig for TencentAnthropicMessagesConfig {
         headers: Headers,
         api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         _params: &LitellmParams,
         env: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -229,7 +230,14 @@ mod tests {
                 .flatten()
         };
         let environment = TENCENT_MESSAGES_CONFIG
-            .validate_environment(vec![], key, "test-model", &Default::default(), &env)
+            .validate_environment(
+                vec![],
+                key,
+                "test-model",
+                &MessagesOptionalParams::default(),
+                &Default::default(),
+                &env,
+            )
             .unwrap();
         assert!(matches!(
             environment.auth,

litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs

@@ -114,6 +114,7 @@ impl BaseMessagesConfig for VertexAiPartnerModelsAnthropicMessagesConfig {
         headers: Headers,
         _api_key: Option<&str>,
         _model: &str,
+        _optional_params: &MessagesOptionalParams,
         litellm_params: &LitellmParams,
         _env_lookup: &dyn Fn(&str) -> Option<String>,
     ) -> Result<ValidatedEnvironment, Error> {
@@ -439,6 +440,7 @@ mod tests {
                     .collect(),
                 Some("ignored-api-key"),
                 "claude-sonnet-4-5",
+                &MessagesOptionalParams::default(),
                 &params(Some("proj"), None),
                 &no_env,
             )

litellm-rust/crates/python-bridge/src/routes/responses/host.rs

@@ -1,22 +1,41 @@
 use std::convert::Infallible;
 
 use crate::routes::inference::InferenceHost;
-use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned};
-use litellm_inference_responses::{Error, route::Responses, types::ResponsesCall};
+use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
+use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned, from_py};
+use litellm_inference_responses::{
+    Error,
+    ids::{self, IdScope},
+    route::Responses,
+    types::ResponsesCall,
+};
 use pyo3::{
     gc::{PyTraverseError, PyVisit},
     prelude::*,
     types::PyDict,
 };
 
-pub(super) struct ResponsesPythonHost(pub InferenceHost);
+pub(super) struct ResponsesPythonHost {
+    host: InferenceHost,
+    ids: IdScope,
+}
+
+impl ResponsesPythonHost {
+    pub fn new(host: InferenceHost) -> Self {
+        Self {
+            host,
+            ids: IdScope::default(),
+        }
+    }
+}
 
 pub(super) fn project(
     host: &mut InferenceHost,
     py: Python<'_>,
     arguments: &Bound<'_, PyDict>,
 ) -> PyResult<ResponsesCall> {
-    let call = host.project(py, arguments, "input")?;
+    let mut call = host.project(py, arguments, "input")?;
+    ids::restore_input(&mut call.input);
     let optional_params = call
         .params
         .into_iter()
@@ -50,6 +69,39 @@ pub(super) fn project(
     })
 }
 
+/// The provider and deployment Python encodes into the ids it returns: the resolved provider and
+/// `litellm_metadata.model_info.id`
+fn id_scope(
+    host: &InferenceHost,
+    py: Python<'_>,
+    arguments: &Bound<'_, PyDict>,
+    call: &ResponsesCall,
+) -> PyResult<IdScope> {
+    let metadata = host.argument(py, arguments, "litellm_metadata")?;
+    let encrypted_content_affinity = match &metadata {
+        Some(metadata) => metadata
+            .call_method1("get", ("encrypted_content_affinity_enabled",))?
+            .is_truthy()?,
+        None => false,
+    };
+    let metadata = metadata
+        .map(|value| from_py::<serde_json::Value>(&value))
+        .transpose()?
+        .unwrap_or_default();
+    let model_id = match metadata.pointer("/model_info/id") {
+        None | Some(serde_json::Value::Null) => None,
+        Some(serde_json::Value::String(id)) => Some(id.clone()),
+        Some(other) => Some(other.to_string()),
+    };
+    Ok(IdScope {
+        provider: get_custom_llm_provider(&call.model, call.custom_llm_provider.as_deref())
+            .map(|resolved| resolved.custom_llm_provider.to_owned())
+            .or_else(|| call.custom_llm_provider.clone()),
+        model_id,
+        encrypted_content_affinity,
+    })
+}
+
 impl PythonBinding for ResponsesPythonHost {
     type Protocol = Responses;
     type Failure = PyErr;
@@ -59,7 +111,8 @@ impl PythonBinding for ResponsesPythonHost {
         py: Python<'_>,
         arguments: &Bound<'_, PyDict>,
     ) -> Result<ResponsesCall, InvokeError<Error>> {
-        let call = project(&mut self.0, py, arguments).map_err(InvokeError::Python)?;
+        let call = project(&mut self.host, py, arguments).map_err(InvokeError::Python)?;
+        self.ids = id_scope(&self.host, py, arguments, &call).map_err(InvokeError::Python)?;
         if call
             .optional_params
             .get("stream")
@@ -77,7 +130,9 @@ impl PythonBinding for ResponsesPythonHost {
         py: Python<'_>,
         response: <Responses as litellm_host::protocol::Protocol>::Response,
     ) -> PyResult<Py<PyAny>> {
-        self.0.response(py, &response)
+        let mut response = response;
+        ids::encode_response(&mut response, &self.ids);
+        self.host.response(py, &response)
     }
 
     fn encode_stream_head(
@@ -103,7 +158,7 @@ impl PythonBinding for ResponsesPythonHost {
     }
 
     fn map_error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
-        self.0.error(py, error)
+        self.host.error(py, error)
     }
     fn host_error(error: &PyErr) -> Error {
         Error::InvalidRequest(error.to_string().into())
@@ -122,9 +177,9 @@ impl PythonHostCalls<Responses> for ResponsesPythonHost {
 
 impl PythonOwned for ResponsesPythonHost {
     fn close(&mut self, _: Python<'_>) {
-        self.0.close();
+        self.host.close();
     }
     fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> {
-        self.0.traverse(visit)
+        self.host.traverse(visit)
     }
 }

litellm-rust/crates/python-bridge/src/routes/responses/mod.rs

@@ -62,7 +62,7 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py
         ));
     }
     let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE);
-    run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost(host))
+    run_inference::<ResponsesRoute, _>(py, call, asynchronous, ResponsesPythonHost::new(host))
 }
 
 #[pyfunction]

litellm/__init__.py

@@ -7,6 +7,7 @@ _apply_process_defaults()
 ### INIT VARIABLES #########################
 import threading
 import os
+import datetime
 
 from collections.abc import Iterable, Mapping, Sequence
 from typing import (
@@ -62,7 +63,6 @@ from litellm.constants import (
     modelscope_models,
     empower_models,
     together_ai_models,
-    baseten_models,
     WANDB_MODELS,
     REPEATED_STREAMING_CHUNK_LIMIT,
     request_timeout,
@@ -628,6 +628,7 @@ bedrock_models: Set[str] = set()
 bedrock_converse_models: Set[str] = set(BEDROCK_CONVERSE_MODELS)
 fal_ai_models: Set[str] = set()
 fireworks_ai_models: Set[str] = set()
+baseten_models: Set[str] = set()
 fireworks_ai_embedding_models: Set[str] = set()
 deepinfra_models: Set[str] = set()
 perplexity_models: Set[str] = set()
@@ -746,6 +747,16 @@ def is_openai_finetune_model(key: str) -> bool:
     return key.startswith("ft:") and not key.count(":") > 1
 
 
+def _is_past_deprecation_date(value: Mapping[str, object]) -> bool:
+    deprecation_date: Final = value.get("deprecation_date")
+    if not isinstance(deprecation_date, str):
+        return False
+    try:
+        return datetime.date.fromisoformat(deprecation_date) <= datetime.date.today()
+    except ValueError:
+        return False
+
+
 def _provider_model_set(key: str, value: Mapping[str, object]) -> Optional[Tuple[Set[str], str]]:
     provider: Final = value.get("litellm_provider")
     if provider == "openai" and not is_openai_finetune_model(key):
@@ -835,6 +846,8 @@ def _provider_model_set(key: str, value: Mapping[str, object]) -> Optional[Tuple
         # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
         if "-to-" not in key and "fireworks-ai-default" not in key:
             return fireworks_ai_models, key
+    if provider == "baseten":
+        return None if _is_past_deprecation_date(value) else (baseten_models, key)
     if provider == "fireworks_ai-embedding-models":
         # ignore the 'up-to', '-to-' model names -> not real models. just for cost tracking based on model params.
         if "-to-" not in key:
@@ -1004,6 +1017,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
     for key, value in tuple(model_cost_map.items()):
         if (membership := _provider_model_set(key, value)) is not None:
             membership[0].add(membership[1])
+        elif value.get("litellm_provider") == "baseten":
+            baseten_models.discard(key)
 
 
 def add_known_models(model_cost_map: Optional[Dict] = None):

litellm/constants.py

@@ -40,6 +40,14 @@ class ServerStreamingClassification(str, Enum):
 SERVER_STREAMING_CLASSIFICATION_MARKER: Final = ServerStreamingClassification.MARKER
 DEFER_PYDANTIC_BUILD: Final = os.getenv("DEFER_PYDANTIC_BUILD", "true") in ("true", "1", "on")
 DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm"))
+KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS: Final = get_env_float(
+    "KUBERNETES_POD_DISCOVERY_REFRESH_INTERVAL_SECONDS", 5.0
+)
+KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS: Final = get_env_float(
+    "KUBERNETES_POD_DISCOVERY_IDLE_EVICTION_SECONDS", 300.0
+)
+KUBERNETES_POD_DISCOVERY_DNS_TIMEOUT_SECONDS: Final = 2.0
+KUBERNETES_POD_ROUTING_KEY: Final = "kubernetes_pod_routing"
 AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview"))
 AZURE_OPENAI_AUDIO_PROVIDERS: Final = frozenset({"azure", "azure_ai"})
 ROUTER_MAX_FALLBACKS: Final = get_env_int("ROUTER_MAX_FALLBACKS", 5)
@@ -1256,14 +1264,6 @@ together_ai_models: Final[set] = set(
 # supports all together ai models, just pass in the model id e.g. completion(model="together_computer/replit_code_3b",...)
 
 
-baseten_models: Final[set] = set(
-    [
-        "qvv0xeq",
-        "q841o8w",
-        "31dxrj3",
-    ]
-)  # FALCON 7B  # WizardLM  # Mosaic ML
-
 featherless_ai_models: Final[set] = set(
     [
         "featherless-ai/Qwerky-72B",

litellm/cost_calculator.py

@@ -3116,14 +3116,19 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
         return response_ids[-1] if response_ids else None
 
     @staticmethod
-    def create_logging_object(results: Sequence[Mapping[str, object]]) -> LiteLLMRealtimeStreamLoggingObject:
+    def create_logging_object(
+        results: Sequence[Mapping[str, object]], litellm_model_name: str | None = None
+    ) -> LiteLLMRealtimeStreamLoggingObject:
         tier_partition: Final = ResponsesWebSocketTokenUsageProcessor.partition_results_by_service_tier(results)
-        return LiteLLMRealtimeStreamLoggingObject(
+        logging_object: Final = LiteLLMRealtimeStreamLoggingObject(
             id=ResponsesWebSocketTokenUsageProcessor.response_id_from_responses_ws_results(results),
             usage=ResponsesWebSocketTokenUsageProcessor.collect_and_combine_usage_from_responses_ws_results(results),
             results=cast(OpenAIRealtimeStreamList, list(results)),  # cast-ok: raw WS frames, unvalidated field
             service_tier=next(iter(tier_partition)) if len(tier_partition) == 1 else None,
         )
+        if litellm_model_name is not None:
+            logging_object.hidden_params = {"litellm_model_name": litellm_model_name}
+        return logging_object
 
 
 _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"

litellm/decisions/call.py

@@ -1,3 +1,4 @@
+import json
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from functools import partial
@@ -6,6 +7,7 @@ from typing import Final, Literal, TypeAlias
 
 import httpx
 from pydantic import BaseModel, ConfigDict, StrictBool, TypeAdapter, ValidationError
+from pydantic_core import to_jsonable_python
 from typing_extensions import assert_never
 
 import litellm
@@ -56,6 +58,7 @@ from litellm.types.decisions import (
     OpenAIDecisionResponse,
     UnsupportedDecisionsRequest,
 )
+from litellm.types.llms.openai import ChatCompletionUserMessage
 from litellm.types.utils import LlmProviders, ProviderSpecificHeader
 from litellm.utils import ProviderConfigManager, get_model_cost_key
 
@@ -415,6 +418,30 @@ def _outbound_headers(
     return MappingProxyType({**(extra_headers or {}), **forwarded})
 
 
+_LOGGED_FIELDS_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+
+
+def _jsonable_fallback(value: object) -> object:
+    return dict(_LOGGED_FIELDS_ADAPTER.validate_python(value)) if isinstance(value, Mapping) else str(value)
+
+
+def _logged_request_message(caller_request: object) -> ChatCompletionUserMessage:
+    caller_fields: Final = _LOGGED_FIELDS_ADAPTER.validate_python(
+        to_jsonable_python(caller_request, fallback=_jsonable_fallback)
+    )
+    sent_fields: Final = json.dumps(
+        dict((name, value) for name, value in caller_fields.items() if value not in (None, [], {}))
+    )
+    return ChatCompletionUserMessage(role="user", content=sent_fields)
+
+
+def log_caller_request(kwargs: Mapping[str, object], caller_request: object) -> None:
+    """Record every request field the caller sent as the logged `messages`, before any validation."""
+    logging_obj: Final = kwargs.get("litellm_logging_obj")
+    if isinstance(logging_obj, LiteLLMLoggingObj):
+        logging_obj.update_messages([_logged_request_message(caller_request)])
+
+
 def prepare_call(
     *,
     model: str,

litellm/decisions/main.py

@@ -17,6 +17,7 @@ from litellm.decisions.call import (
     SystemOneQuestions,
     SystemOneRequestFields,
     asend,
+    log_caller_request,
     prepare_call,
     reject_images_next_to_input,
     send,
@@ -78,6 +79,16 @@ def _prepare_decisions_call(
     extra_headers: Mapping[str, str] | None,
     kwargs: Mapping[str, object],
 ) -> DecisionsCall:
+    log_caller_request(
+        kwargs,
+        {
+            "input": decision_input,
+            "questions": questions,
+            "audio": audio,
+            "videos": videos,
+            **{key: kwargs[key] for key in _SYSTEMONE_FIELDS if key in kwargs},
+        },
+    )
     return prepare_call(
         model=model,
         fields=_request_fields(model, decision_input, questions, audio, videos, kwargs),

litellm/google_genai/main.py

@@ -16,6 +16,7 @@ from litellm.llms.base_llm.google_genai.transformation import (
     BaseGoogleGenAIGenerateContentConfig,
 )
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
+from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import CallTypes
@@ -194,12 +195,14 @@ class GenerateContentHelper:
         if litellm_logging_obj is None:
             raise ValueError("litellm_logging_obj is required, but got None")
 
+        vertex_location: Final = VertexBase.explicit_vertex_ai_location(dict(litellm_params))
         litellm_logging_obj.update_from_kwargs(
             kwargs=kwargs,
             model=model,
             optional_params=dict(generate_content_config_dict),
             litellm_params={
                 "litellm_call_id": litellm_call_id,
+                **({"vertex_location": vertex_location} if vertex_location else {}),
             },
             custom_llm_provider=custom_llm_provider,
         )

litellm/integrations/custom_logger.py

@@ -912,9 +912,11 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         This is useful for logging payloads that contain sensitive information.
         """
         import litellm
-        from litellm import Choices, Message, ModelResponse
         from litellm.litellm_core_utils.classifier_logging import CLASSIFIER_AUDIT_FIELDS
-        from litellm.litellm_core_utils.redact_messages import redacted_litellm_params
+        from litellm.litellm_core_utils.redact_messages import (
+            redacted_litellm_params,
+            redacted_standard_logging_payload,
+        )
 
         turn_off_message_logging: Final[bool] = getattr(self, "turn_off_message_logging", False)
         excluded_fields: Final[list[str] | None] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
@@ -940,41 +942,14 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
             if key not in (excluded_fields or ()) and not (turn_off_message_logging and key in CLASSIFIER_AUDIT_FIELDS)
         }
 
-        # Handle turn_off_message_logging - redact messages and responses (if not already excluded)
-        if turn_off_message_logging and not self.redacts_messages_itself():
-            redacted_str: Final = "redacted-by-litellm"
-
-            if "messages" not in (excluded_fields or ()) and standard_logging_object_copy.get("messages") is not None:
-                standard_logging_object_copy["messages"] = [Message(content=redacted_str).model_dump()]
-
-            if "response" not in (excluded_fields or ()) and standard_logging_object_copy.get("response") is not None:
-                response: Final = standard_logging_object_copy["response"]
-                # Check if this is a ResponsesAPIResponse (has "output" field)
-                if isinstance(response, dict) and "output" in response:
-                    # Make a copy to avoid modifying the original
-                    from copy import deepcopy
-
-                    response_copy: Final = deepcopy(response)
-                    # Redact content in output array
-                    if isinstance(response_copy.get("output"), list):
-                        for output_item in response_copy["output"]:
-                            if isinstance(output_item, dict) and "content" in output_item:
-                                if isinstance(output_item["content"], list):
-                                    # Redact text in content items
-                                    for content_item in output_item["content"]:
-                                        if isinstance(content_item, dict) and "text" in content_item:
-                                            content_item["text"] = redacted_str
-                    standard_logging_object_copy["response"] = response_copy
-                else:
-                    # Standard ModelResponse format
-                    model_response: Final = ModelResponse(choices=[Choices(message=Message(content=redacted_str))])
-                    model_response_dict: Final = model_response.model_dump()
-                    standard_logging_object_copy["response"] = model_response_dict
-
         return {
             **model_call_details,
             **redacted_params,
-            "standard_logging_object": standard_logging_object_copy,
+            "standard_logging_object": (
+                redacted_standard_logging_payload(standard_logging_object_copy)
+                if turn_off_message_logging and not self.redacts_messages_itself()
+                else standard_logging_object_copy
+            ),
         }
 
     async def get_proxy_server_request_from_cold_storage_with_object_key(

litellm/integrations/opentelemetry.py

@@ -13,6 +13,7 @@ from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.constants import KUBERNETES_POD_ROUTING_KEY
 from litellm.integrations._types.open_inference import (
     OpenInferenceSpanKindValues,
 )
@@ -50,6 +51,7 @@ from litellm.types.utils import (
     Function,
     LLMResponseTypes,
     StandardCallbackDynamicParams,
+    StandardLoggingMetadata,
     StandardLoggingPayload,
 )
 
@@ -2420,9 +2422,15 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             #############################################
             ############ LLM CALL METADATA ##############
             #############################################
-            metadata: Final = standard_logging_payload["metadata"]
+            metadata: Final = cast(  # cast-ok: payload metadata follows the standard logging contract
+                StandardLoggingMetadata, standard_logging_payload["metadata"]
+            )
             for key, value in metadata.items():
-                self.safe_set_attribute(span=span, key=f"metadata.{key}", value=value)
+                self.safe_set_attribute(
+                    span=span,
+                    key=f"metadata.{key}",
+                    value=safe_dumps(value) if key == KUBERNETES_POD_ROUTING_KEY and value is not None else value,
+                )
             decision: Final = metadata.get("routing_decision")
             span.set_attributes(routing_decision_attributes(decision))
 

litellm/integrations/prometheus.py

@@ -47,9 +47,11 @@ from litellm.litellm_core_utils.core_helpers import (
     get_litellm_metadata_from_kwargs,
     get_metadata_variable_name_from_kwargs,
 )
+from litellm.litellm_core_utils.get_llm_provider_logic import is_registered_custom_provider
 from litellm.litellm_core_utils.service_tier_utils import (
     get_service_tier_from_standard_logging_payload,
 )
+from litellm.llms.openai_like.json_loader import JSONProviderRegistry
 from litellm.models.end_user import LiteLLM_EndUserTable
 from litellm.models.team import LiteLLM_TeamTable
 from litellm.models.user import LiteLLM_UserTable
@@ -60,6 +62,7 @@ from litellm.repositories.organization_repository import OrganizationRepository
 from litellm.repositories.table_repositories import EndUserRepository
 from litellm.repositories.team_repository import TeamRepository
 from litellm.repositories.user_repository import UserRepository
+from litellm.router_utils.router_callbacks.track_deployment_metrics import find_deployment_metadata
 from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.integrations.prometheus import *
 from litellm.types.integrations.prometheus import (
@@ -260,6 +263,26 @@ _POSITIVE_SERIES_TTL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, F
 _SERIES_CLEANUP_INTERVAL: Final[TypeAdapter[float]] = TypeAdapter(Annotated[float, Field(ge=0)])
 _DEFAULT_SERIES_CLEANUP_INTERVAL_SECONDS: Final = 60.0
 _ADDITIONAL_HEADERS: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object])
+_LOGGED_FIELDS: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+_NO_LOGGED_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
+
+
+def _logged_fields(value: object) -> Mapping[str, object]:
+    return _LOGGED_FIELDS.validate_python(value) if isinstance(value, Mapping) else _NO_LOGGED_FIELDS
+
+
+def _is_llm_provider(name: str) -> bool:
+    routable: Final = (
+        name in litellm.provider_list or JSONProviderRegistry.exists(name) or is_registered_custom_provider(name)
+    )
+    return routable and name not in litellm._known_custom_logger_compatible_callbacks  # pyright: ignore[reportPrivateUsage]  # the router treats these prefixes as prompt-management deployments
+
+
+def _deployment_model_provider(deployment: object) -> str:
+    if not isinstance(deployment, str):
+        return ""
+    prefix, separator, _ = deployment.partition("/")
+    return prefix if separator and _is_llm_provider(prefix) else ""
 
 
 def _number_or_none(value: object, limit: TypeAdapter[_SeriesLimitT]) -> _SeriesLimitT | None:
@@ -2846,6 +2869,25 @@ class PrometheusLogger(CustomLogger):
             )
             return None
 
+    @staticmethod
+    def _extract_deployment_api_provider(request_kwargs: Mapping[str, object]) -> str:
+        """
+        Provider of the deployment a failed call was routed to: the resolved
+        ``custom_llm_provider`` on ``litellm_params``, then the one recorded on
+        ``standard_logging_object``, then the provider prefix of the deployment
+        model the router wrote into the request metadata (``azure/my-gpt`` ->
+        ``azure``) when that prefix is an LLM provider the router accepts.
+        Returns an empty string otherwise.
+        """
+        litellm_params: Final = _logged_fields(request_kwargs.get("litellm_params"))
+        provider: Final = litellm_params.get("custom_llm_provider") or _logged_fields(
+            request_kwargs.get("standard_logging_object")
+        ).get("custom_llm_provider")
+        if isinstance(provider, str) and provider:
+            return provider
+        deployment_metadata: Final = find_deployment_metadata(litellm_params)
+        return _deployment_model_provider(deployment_metadata.get("deployment") if deployment_metadata else None)
+
     @staticmethod
     def _extract_api_provider_from_exception(exception: Exception) -> str | None:
         if not isinstance(exception, litellm.exceptions.RateLimitError):
@@ -3124,7 +3166,9 @@ class PrometheusLogger(CustomLogger):
                 label_litellm_model_name = litellm_model_name
                 label_model_id = model_id
                 label_api_base = api_base
-                label_api_provider = llm_provider
+                label_api_provider = self._extract_deployment_api_provider(
+                    _LOGGED_FIELDS.validate_python(request_kwargs)
+                )
                 label_requested_model = model_group or litellm_model_name
                 label_model_group = model_group
             else:

litellm/litellm_core_utils/core_helpers.py

@@ -10,14 +10,16 @@ from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast, runtime_c
 import httpx
 from pydantic import TypeAdapter, ValidationError
 
+import litellm
 from litellm._logging import verbose_logger
+from litellm.constants import KUBERNETES_POD_ROUTING_KEY
 from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
 from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
-    from litellm.types.utils import ModelResponseStream
+    from litellm.types.utils import ModelResponseStream, StandardLoggingKubernetesPodRouting
 
     Span = _Span | Any
 else:
@@ -92,6 +94,10 @@ def normalize_drop_params(value: object) -> bool | None:
         return None
 
 
+def drops_unsupported_params(drop_params: object) -> bool:
+    return litellm.drop_params is True or normalize_drop_params(drop_params) is True
+
+
 def drop_params_flag(value: object, source: str, logger: logging.Logger) -> bool:
     normalized: Final = normalize_drop_params(value)
     if normalized is None and value is not None:
@@ -385,6 +391,54 @@ def proxy_stamped_used_client_oauth_token(metadata: object, litellm_params: Mapp
     return metadata.get("used_client_oauth_token") if isinstance(metadata, Mapping) else None
 
 
+def proxy_stamped_kubernetes_pod_routing(
+    metadata: object,
+    litellm_params: Mapping[str, object] | None,
+) -> "StandardLoggingKubernetesPodRouting | None":
+    litellm_metadata: Final = litellm_params.get("litellm_metadata") if litellm_params is not None else None
+    litellm_routing: Final = _standard_logging_kubernetes_pod_routing(litellm_metadata)
+    if litellm_routing is not None:
+        return litellm_routing
+    return _standard_logging_kubernetes_pod_routing(metadata)
+
+
+def _standard_logging_kubernetes_pod_routing(
+    metadata: object,
+) -> "StandardLoggingKubernetesPodRouting | None":
+    if not isinstance(metadata, Mapping):
+        return None
+    metadata_mapping: Final = cast(  # cast-ok: routing metadata is a string-keyed mapping
+        Mapping[str, object], metadata
+    )
+    routing: Final = metadata_mapping.get(KUBERNETES_POD_ROUTING_KEY)
+    if not isinstance(routing, Mapping):
+        return None
+    routing_mapping: Final = cast(  # cast-ok: pod routing records use string keys
+        Mapping[str, object], routing
+    )
+    service_host: Final = routing_mapping.get("service_host")
+    pod_ip: Final = routing_mapping.get("pod_ip")
+    pod_count: Final = routing_mapping.get("pod_count")
+    selection: Final = routing_mapping.get("selection")
+    if (
+        not isinstance(service_host, str)
+        or not isinstance(pod_ip, str)
+        or not isinstance(pod_count, int)
+        or isinstance(pod_count, bool)
+        or pod_count < 1
+    ):
+        return None
+    if selection in ("round_robin", "session_affinity", "session_affinity_retry"):
+        routing_record: Final[StandardLoggingKubernetesPodRouting] = {
+            "service_host": service_host,
+            "pod_ip": pod_ip,
+            "pod_count": pod_count,
+            "selection": selection,
+        }
+        return routing_record
+    return None
+
+
 def get_litellm_metadata_from_kwargs(kwargs: dict):
     """
     Helper to get litellm metadata from all litellm request kwargs

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -2551,7 +2551,8 @@ def exception_type(
             ################################################################################
 
             if (
-                "Request Timeout Error" in error_str
+                isinstance(original_exception, httpx.TimeoutException)
+                or "Request Timeout Error" in error_str
                 or "Request timed out" in error_str
                 or "Timed out generating response" in error_str
                 or "The read operation timed out" in error_str
@@ -2559,7 +2560,7 @@ def exception_type(
                 exception_mapping_worked = True
 
                 raise Timeout(
-                    message=f"APITimeoutError - Request timed out. Error_str: {error_str}",
+                    message=f"APITimeoutError - Request timed out. Error_str: {error_str or type(mappable_exception).__name__}",
                     model=model,
                     llm_provider=custom_llm_provider,
                     litellm_debug_info=extra_information,

litellm/litellm_core_utils/get_litellm_params.py

@@ -142,14 +142,12 @@ def get_base_model_from_litellm_call_metadata(
     if metadata is None:
         return None
     model_info: Final = metadata.get("model_info")
-    if model_info:
-        model_info_mapping: Final = cast(  # cast-ok: model metadata is caller-provided and preserves its mapping shape
-            Mapping[str, object], model_info
-        )
-        return cast(  # cast-ok: model metadata values are caller-provided
-            str | None, model_info_mapping.get("base_model")
-        )
-    return None
+    if not model_info:
+        return None
+    base_model: Final = cast(  # cast-ok: model metadata is caller-provided and preserves its mapping shape
+        Mapping[str, object], model_info
+    ).get("base_model")
+    return base_model if isinstance(base_model, str) else None
 
 
 _get_base_model_from_litellm_call_metadata = get_base_model_from_litellm_call_metadata

litellm/litellm_core_utils/litellm_logging.py

@@ -43,6 +43,7 @@ from litellm.constants import (
     DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
     EMPTY_MAPPING,
     INTERNAL_CALLBACK_NAMES,
+    KUBERNETES_POD_ROUTING_KEY,
     PROVIDER_REQUEST_ID_HEADERS,
     REDACTED_BY_LITELLM,
 )
@@ -75,6 +76,7 @@ from litellm.litellm_core_utils.classifier_logging import (
 from litellm.litellm_core_utils.core_helpers import (
     get_provider_response_headers_from_hidden_params,
     is_expected_client_error,
+    proxy_stamped_kubernetes_pod_routing,
     proxy_stamped_used_client_oauth_token,
     reconstruct_model_name,
     set_response_cost_in_hidden_params,
@@ -149,6 +151,7 @@ from litellm.types.llms.openai import (
 from litellm.types.mcp import MCPPostCallResponseObject
 from litellm.types.prompts.init_prompts import PromptSpec
 from litellm.types.rerank import RerankResponse
+from litellm.types.router import RouterRateLimitError
 from litellm.types.utils import (
     DEPLOYMENT_SCOPED_PRICING_FIELDS,
     CachingDetails,
@@ -298,7 +301,9 @@ else:
     _GENERIC_API_LOGGER_CLS: Final = GenericAPILogger
 _in_memory_loggers: Final[list[CustomLogger]] = []
 
-_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(("used_client_oauth_token", "usage_object"))
+_STANDARD_LOGGING_METADATA_RESOLVED_KEYS: Final[frozenset[str]] = frozenset(
+    ("used_client_oauth_token", "usage_object", KUBERNETES_POD_ROUTING_KEY)
+)
 _STANDARD_LOGGING_METADATA_KEYS: Final[frozenset[str]] = (
     frozenset(StandardLoggingMetadata.__annotations__.keys()) - _STANDARD_LOGGING_METADATA_RESOLVED_KEYS
 )
@@ -634,15 +639,15 @@ class Logging(LiteLLMLoggingBaseClass):
         self,
         model: str,
         messages,
-        stream,
+        stream: bool | None,
         call_type,
         start_time: datetime.datetime,
         litellm_call_id: str,
         function_id: str,
         litellm_trace_id: str | None = None,
         dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = None,
-        dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = None,
-        dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = None,
+        dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
+        dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = None,
         dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
         dynamic_async_failure_callbacks: list[str | Callable | CustomLogger] | None = None,
         applied_guardrails: list[str] | None = None,
@@ -718,8 +723,10 @@ class Logging(LiteLLMLoggingBaseClass):
 
         # Initialize dynamic callbacks
         self.dynamic_input_callbacks: list[str | Callable | CustomLogger] | None = dynamic_input_callbacks
-        self.dynamic_success_callbacks: list[str | Callable | CustomLogger] | None = dynamic_success_callbacks
-        self.dynamic_async_success_callbacks: list[str | Callable | CustomLogger] | None = (
+        self.dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
+            dynamic_success_callbacks
+        )
+        self.dynamic_async_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None = (
             dynamic_async_success_callbacks
         )
         self.dynamic_failure_callbacks: list[str | Callable | CustomLogger] | None = dynamic_failure_callbacks
@@ -786,6 +793,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self.defer_async_logging: bool = False
         self.enqueue_deferred_logging: Callable[[], None] | None = None
         self._async_success_scheduled: bool = False
+        self._sync_success_scheduled: bool = False
         self.on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
         self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
 
@@ -986,6 +994,16 @@ class Logging(LiteLLMLoggingBaseClass):
                 return model_id
         return None
 
+    def _routed_deployment_model(self) -> str | None:
+        litellm_params: Final = self._model_call_details_view.get("litellm_params")
+        metadata: Final = (
+            StandardLoggingPayloadSetup.merge_litellm_metadata(_LITELLM_PARAMS.validate_python(litellm_params))
+            if isinstance(litellm_params, dict)
+            else {}
+        )
+        deployment: Final = metadata.get("deployment")
+        return deployment if isinstance(deployment, str) else None
+
     def get_deployment_model_for_cost(self) -> str | None:
         """The provider-qualified model to price against.
 
@@ -2253,6 +2271,7 @@ class Logging(LiteLLMLoggingBaseClass):
             and litellm_params.get(CallTypes.agenerate_content.value, False) is not True
             and litellm_params.get(CallTypes.agenerate_content_stream.value, False) is not True
             and litellm_params.get(CallTypes.arealtime.value, False) is not True
+            and litellm_params.get(CallTypes.aresponses_websocket.value, False) is not True
         )
 
     _is_sync_litellm_request = is_sync_litellm_request
@@ -2318,7 +2337,7 @@ class Logging(LiteLLMLoggingBaseClass):
             **kwargs,
         )
 
-        if not self._should_run_sync_callbacks_for_async_calls():
+        if not self.stream or not self._should_run_sync_callbacks_for_async_calls():
             return
 
         executor.submit(
@@ -2446,7 +2465,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self,
         event_type: Literal["async_success", "sync_success", "async_failure", "sync_failure"],
     ) -> None:
-        if self.stream is not None and self.stream is True and event_type in ["async_success", "sync_success"]:
+        if self.stream is True and event_type in ["async_success", "sync_success"]:
             """
             Ignore check on stream, as there can be multiple chunks
             """
@@ -2508,7 +2527,8 @@ class Logging(LiteLLMLoggingBaseClass):
 
         elif self.call_type == CallTypes.aresponses_websocket.value and isinstance(result, list):  # pyright: ignore[reportUnknownMemberType]  # Logging.call_type is untyped
             logging_result = ResponsesWebSocketTokenUsageProcessor.create_logging_object(
-                results=result  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
+                results=result,  # pyright: ignore[reportUnknownArgumentType]  # raw event dicts from the WS stream
+                litellm_model_name=self._routed_deployment_model(),
             )
 
         elif self.relays_llm_passthrough and isinstance(result, Response):
@@ -2943,6 +2963,8 @@ class Logging(LiteLLMLoggingBaseClass):
         discarded as a duplicate of the in-progress create.
         """
         self.model_call_details.pop("has_logged_async_success", None)
+        self.model_call_details.pop("has_logged_sync_success", None)
+        self._sync_success_scheduled = False
         litellm_params = self.model_call_details.get("litellm_params")
         if not isinstance(litellm_params, dict):
             return
@@ -3131,6 +3153,11 @@ class Logging(LiteLLMLoggingBaseClass):
                 dynamic_success_callbacks=self.dynamic_success_callbacks,
                 global_callbacks=litellm.success_callback,
             )
+            delivery_callbacks: Final = (
+                self._get_completed_success_callbacks(asynchronous=False)
+                if for_async_request and not self.stream
+                else callbacks
+            )
 
             ## REDACT MESSAGES ##
             result = redact_message_input_output_from_logging(
@@ -3138,7 +3165,8 @@ class Logging(LiteLLMLoggingBaseClass):
                 result=result,
             )
             ## LOGGING HOOK ##
-            for callback in callbacks:
+            hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
+            for callback in hook_callbacks:
                 if isinstance(callback, CustomGuardrail):
                     from litellm.types.guardrails import GuardrailEventHooks
 
@@ -3168,7 +3196,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
             call_details_view: Final = MappingProxyType[str, object](call_details)
             self.mark_logging_complete(event_type="sync_success")
-            for callback in callbacks:
+            for callback in delivery_callbacks:
                 try:
                     should_run = self.should_run_callback(
                         callback=callback,
@@ -3440,7 +3468,12 @@ class Logging(LiteLLMLoggingBaseClass):
                             )
                     if (
                         isinstance(callback, CustomLogger)
-                        and is_sync_request
+                        and (is_sync_request or (for_async_request and not self.stream))
+                        and not (
+                            for_async_request
+                            and self.stream
+                            and callback in (self.dynamic_async_success_callbacks or ())
+                        )
                         and self.call_type
                         != CallTypes.pass_through.value  # pass-through endpoints call async_log_success_event
                     ):  # custom logger class
@@ -3457,7 +3490,7 @@ class Logging(LiteLLMLoggingBaseClass):
                                 result = call_details_view["complete_response"]
 
                             callback.log_success_event(
-                                kwargs=call_details,
+                                kwargs=callback.redact_standard_logging_payload_from_model_call_details(call_details),
                                 response_obj=result,
                                 start_time=start_time,
                                 end_time=end_time,
@@ -3699,6 +3732,9 @@ class Logging(LiteLLMLoggingBaseClass):
             dynamic_success_callbacks=self.dynamic_async_success_callbacks,
             global_callbacks=litellm._async_success_callback,
         )
+        delivery_callbacks: Final = (
+            self._get_completed_success_callbacks(asynchronous=True) if not self.stream else callbacks
+        )
 
         result = redact_message_input_output_from_logging(
             model_call_details=(self.model_call_details if hasattr(self, "model_call_details") else {}),
@@ -3707,7 +3743,8 @@ class Logging(LiteLLMLoggingBaseClass):
 
         ## LOGGING HOOK ##
 
-        for callback in callbacks:
+        hook_callbacks: Final = tuple(dict.fromkeys((*callbacks, *delivery_callbacks)))
+        for callback in hook_callbacks:
             try:
                 if isinstance(callback, CustomGuardrail):
                     from litellm.types.guardrails import GuardrailEventHooks
@@ -3745,7 +3782,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
         self.mark_logging_complete(event_type="async_success")
 
-        for callback in callbacks:
+        for callback in delivery_callbacks:
             # check if callback can run for this request
             litellm_params = self.model_call_details.get("litellm_params", {})
             should_run = self.should_run_callback(
@@ -3880,6 +3917,13 @@ class Logging(LiteLLMLoggingBaseClass):
                     traceback.format_exc(),
                 )
                 self._handle_callback_failure(callback=callback)
+        if not self.stream:
+            try:
+                self.handle_sync_success_callbacks_for_async_calls(
+                    result=result, start_time=start_time, end_time=end_time, cache_hit=cache_hit
+                )
+            except RuntimeError:
+                verbose_logger.exception("Could not schedule synchronous success callbacks")
         await flush_post_call_redis_batches()
 
     def _handle_callback_failure(self, callback: object):
@@ -4303,6 +4347,10 @@ class Logging(LiteLLMLoggingBaseClass):
         if self._should_run_sync_callbacks_for_async_calls() is False:
             return
 
+        if not self.stream and self._sync_success_scheduled:
+            return
+        if not self.stream:
+            self._sync_success_scheduled = True
         executor.submit(
             self.success_handler,
             result,
@@ -4321,11 +4369,57 @@ class Logging(LiteLLMLoggingBaseClass):
     ) -> None:
         executor.submit(self.success_handler, result, start_time, end_time, cache_hit)
 
+    def _get_completed_success_callbacks(
+        self, *, asynchronous: bool
+    ) -> tuple[str | Callable[..., object] | CustomLogger, ...]:
+        sync_callbacks: Final = self.get_combined_callback_list(
+            self.dynamic_success_callbacks, litellm.success_callback
+        )
+        async_callbacks: Final = self.get_combined_callback_list(
+            self.dynamic_async_success_callbacks,
+            litellm._async_success_callback,  # pyright: ignore[reportPrivateUsage]  # shared logging registry
+        )
+        legacy_sync_callbacks: Final = self._remove_internal_litellm_callbacks(
+            self._remove_internal_custom_logger_callbacks(sync_callbacks)
+        )
+        selected: Final = async_callbacks if asynchronous else sync_callbacks
+        other: Final = sync_callbacks if asynchronous else async_callbacks
+        return tuple(
+            callback
+            for callback in self.get_combined_callback_list(selected, other)
+            if (
+                (
+                    (
+                        getattr(callback.async_log_success_event, "__func__", callback.async_log_success_event)
+                        is not CustomLogger.async_log_success_event
+                    )
+                    == asynchronous
+                    and (
+                        asynchronous
+                        or getattr(callback.log_success_event, "__func__", callback.log_success_event)
+                        is not CustomLogger.log_success_event
+                    )
+                )
+                if isinstance(callback, CustomLogger)
+                else callback in selected
+                and (
+                    asynchronous
+                    or (
+                        callback not in async_callbacks
+                        and callback in legacy_sync_callbacks
+                        and not (isinstance(callback, str) and callback in INTERNAL_CALLBACK_NAMES)
+                    )
+                )
+            )
+        )
+
     def _should_run_sync_callbacks_for_async_calls(self) -> bool:
         """
         Returns:
             - bool: True if sync callbacks should be run for async calls. eg. `langfuse`, `s3`
         """
+        if not self.stream:
+            return bool(self._get_completed_success_callbacks(asynchronous=False))
         _combined_sync_callbacks: Final = self.get_combined_callback_list(
             dynamic_success_callbacks=self.dynamic_success_callbacks,
             global_callbacks=litellm.success_callback,
@@ -4355,7 +4449,11 @@ class Logging(LiteLLMLoggingBaseClass):
         _filtered_failure_callbacks = self._remove_internal_litellm_callbacks(_filtered_failure_callbacks)
         return len(_filtered_failure_callbacks) > 0
 
-    def get_combined_callback_list(self, dynamic_success_callbacks: list | None, global_callbacks: list) -> list:
+    def get_combined_callback_list(
+        self,
+        dynamic_success_callbacks: list[str | Callable[..., object] | CustomLogger] | None,
+        global_callbacks: list[str | Callable[..., object] | CustomLogger],
+    ) -> list[str | Callable[..., object] | CustomLogger]:
         if dynamic_success_callbacks is None:
             return list(global_callbacks)
         return list(dict.fromkeys(dynamic_success_callbacks + global_callbacks))
@@ -6182,7 +6280,7 @@ class StandardLoggingPayloadSetup:
     @staticmethod
     def get_standard_logging_metadata(
         metadata: Mapping[str, object] | None,
-        litellm_params: dict | None = None,
+        litellm_params: dict[str, object] | None = None,
         prompt_integration: str | None = None,
         applied_guardrails: list[str] | None = None,
         mcp_tool_call_metadata: StandardLoggingMCPToolCall | None = None,
@@ -6222,6 +6320,8 @@ class StandardLoggingPayloadSetup:
                     prompt_integration=prompt_integration,
                 )
 
+        kubernetes_pod_routing: Final = proxy_stamped_kubernetes_pod_routing(metadata, litellm_params)
+
         # Initialize with default values
         clean_metadata = StandardLoggingMetadata(
             user_api_key_hash=None,
@@ -6258,6 +6358,7 @@ class StandardLoggingPayloadSetup:
             user_api_key_auth_metadata=None,
             team_alias=None,
             team_id=None,
+            kubernetes_pod_routing=kubernetes_pod_routing,
             used_client_oauth_token=resolve_used_client_oauth_token(
                 proxy_stamped_used_client_oauth_token(metadata, litellm_params),
                 custom_llm_provider,
@@ -6370,6 +6471,7 @@ class StandardLoggingPayloadSetup:
         custom_llm_provider: str | None,
         init_response_obj: object,
         api_base: str | None = None,
+        router_model_id: str | None = None,
     ) -> StandardLoggingModelInformation:
         model_cost_name: Final = select_model_name_for_cost_calc(
             model=base_model if custom_pricing else None,
@@ -6377,12 +6479,23 @@ class StandardLoggingPayloadSetup:
             base_model=base_model,
             custom_pricing=custom_pricing,
         )
+        pricing_model_name: Final = (
+            select_model_name_for_cost_calc(
+                model=base_model if custom_pricing else None,
+                completion_response=init_response_obj,
+                base_model=base_model,
+                custom_pricing=custom_pricing,
+                router_model_id=router_model_id,
+            )
+            if router_model_id is not None
+            else model_cost_name
+        )
         if model_cost_name is None:
             model_cost_information = StandardLoggingModelInformation(model_map_key="", model_map_value=None)
         else:
             try:
                 _model_cost_information: Final = litellm.get_model_info(
-                    model=model_cost_name,
+                    model=pricing_model_name or model_cost_name,
                     custom_llm_provider=custom_llm_provider,
                     api_base=api_base,
                 )
@@ -6592,6 +6705,7 @@ class StandardLoggingPayloadSetup:
         rate_limit_category: Final = validate_rate_limit_category(getattr(original_exception, "category", None))
         rate_limit_type: Final = validate_rate_limit_type(getattr(original_exception, "rate_limit_type", None))
         budget_error: Final = original_exception if isinstance(original_exception, BudgetExceededError) else None
+        router_error: Final = original_exception if isinstance(original_exception, RouterRateLimitError) else None
         provider_request_id: Final = _get_provider_request_id(original_exception) if original_exception else None
 
         return StandardLoggingPayloadErrorInformation(
@@ -6607,6 +6721,7 @@ class StandardLoggingPayloadSetup:
             error_budget_entity_id=budget_error.entity_id if budget_error else None,
             error_budget_limit=budget_error.max_budget if budget_error else None,
             error_budget_spend=budget_error.current_cost if budget_error else None,
+            error_cooldown_deployments=router_error.cooldown_deployments if router_error else None,
             normalized_error=normalize_error(original_exception, error_status, error_message),
         )
 
@@ -7090,6 +7205,7 @@ def get_standard_logging_object_payload(
             custom_llm_provider=kwargs.get("custom_llm_provider"),
             init_response_obj=init_response_obj,
             api_base=litellm_params.get("api_base"),
+            router_model_id=logging_obj.get_router_model_id(),
         )
 
         error_information, error_str = StandardLoggingPayloadSetup.get_error_information_for_logging_payload(
@@ -7325,6 +7441,7 @@ def get_standard_logging_metadata(
         user_api_key_auth_metadata=None,
         team_alias=None,
         team_id=None,
+        kubernetes_pod_routing=None,
         used_client_oauth_token=None,
     )
     if isinstance(metadata, dict):
@@ -7398,6 +7515,7 @@ def create_dummy_standard_logging_payload() -> StandardLoggingPayload:
         requester_ip_address="127.0.0.1",
         requester_metadata=None,
         user_api_key_end_user_id="test_end_user",
+        kubernetes_pod_routing=None,
     )
 
     hidden_params: Final = StandardLoggingHiddenParams(

litellm/litellm_core_utils/prompt_templates/chat_template_tools.py

@@ -0,0 +1,37 @@
+import json
+import re
+from collections.abc import Callable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import Final
+
+
+@dataclass(frozen=True, slots=True)
+class ChatTemplateToolSection:
+    preamble_tokens: int
+    implicit_system_turn_tokens: int
+    render_tool_schemas: Callable[[Sequence[Mapping[str, object]]], str]
+
+
+def _deepseek_v4_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
+    return "\n".join(json.dumps(function, ensure_ascii=False) for function in functions)
+
+
+def _qwen3_5_tool_schemas(functions: Sequence[Mapping[str, object]]) -> str:
+    return "".join(
+        "\n" + json.dumps({"type": "function", "function": function}, ensure_ascii=False) for function in functions
+    )
+
+
+_TOOL_SECTIONS: Final = (
+    (
+        re.compile(r"deepseek-v4(?![.\d]|p\d)", re.IGNORECASE),
+        ChatTemplateToolSection(222, 0, _deepseek_v4_tool_schemas),
+    ),
+    (re.compile(r"qwen3[._]?5(?!\d)", re.IGNORECASE), ChatTemplateToolSection(196, 5, _qwen3_5_tool_schemas)),
+)
+
+
+def chat_template_tool_section(model: str | None) -> ChatTemplateToolSection | None:
+    if not model:
+        return None
+    return next((section for pattern, section in _TOOL_SECTIONS if pattern.search(model)), None)

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1669,7 +1669,7 @@ def _replace_invalid_tool_use_id_chars(tool_use_id: str, invalid_chars: re.Patte
     return invalid_chars.sub("_", tool_use_id) or _TOOL_USE_ID_FALLBACK
 
 
-def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
+def sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
     """Anthropic requires tool_use_id to match ^[a-zA-Z0-9_-]+$."""
     return _replace_invalid_tool_use_id_chars(tool_use_id, _ANTHROPIC_TOOL_USE_ID_INVALID_CHARS)
 
@@ -1815,7 +1815,7 @@ def convert_to_anthropic_tool_result(
         tool_message: Final[ChatCompletionToolMessage] = message
         tool_call_id: str = tool_message["tool_call_id"]
         # Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$
-        sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id)
+        sanitized_tool_use_id = sanitize_anthropic_tool_use_id(tool_call_id)
 
         # We can't determine from openai message format whether it's a successful or
         # error call result so default to the successful result template
@@ -1829,7 +1829,7 @@ def convert_to_anthropic_tool_result(
         function_message: Final[ChatCompletionFunctionMessage] = message
         tool_call_id = function_message.get("tool_call_id") or str(uuid.uuid4())
         # Sanitize tool_use_id to match Anthropic's pattern requirement: ^[a-zA-Z0-9_-]+$
-        sanitized_tool_use_id = _sanitize_anthropic_tool_use_id(tool_call_id)
+        sanitized_tool_use_id = sanitize_anthropic_tool_use_id(tool_call_id)
         anthropic_tool_result = AnthropicMessagesToolResultParam(
             type="tool_result",
             tool_use_id=sanitized_tool_use_id,
@@ -2037,7 +2037,7 @@ def convert_to_anthropic_tool_invoke(
         elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids:
             continue
         else:
-            sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id)
+            sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id)
             _anthropic_tool_use_param = AnthropicMessagesToolUseParam(
                 type="tool_use",
                 id=sanitized_tool_id,
@@ -5900,21 +5900,21 @@ def _optional_tool_name(value: object) -> str | None:
 
 
 def _split_tool_call_ids(calls: Sequence[tuple[str | None, int]]) -> tuple[tuple[str | None, ...], ...]:
-    taken: Final = frozenset(_sanitize_anthropic_tool_use_id(call_id) for call_id, _ in calls if call_id)
+    taken: Final = frozenset(sanitize_anthropic_tool_use_id(call_id) for call_id, _ in calls if call_id)
 
     def fresh(call_id: str) -> Iterator[str]:
         return filter(
-            lambda candidate: _sanitize_anthropic_tool_use_id(candidate) not in taken,
+            lambda candidate: sanitize_anthropic_tool_use_id(candidate) not in taken,
             (f"{call_id}__concat_{n}" for n in itertools.count(1)),
         )
 
     suffixes: Final = MappingProxyType(
-        {_sanitize_anthropic_tool_use_id(call_id): fresh(call_id) for call_id, count in calls if call_id and count > 1}
+        {sanitize_anthropic_tool_use_id(call_id): fresh(call_id) for call_id, count in calls if call_id and count > 1}
     )
     return tuple(
         (
             call_id,
-            *(next(suffixes[_sanitize_anthropic_tool_use_id(call_id)]) for _ in range(count - 1)),
+            *(next(suffixes[sanitize_anthropic_tool_use_id(call_id)]) for _ in range(count - 1)),
         )
         if call_id
         else (None,) * count

litellm/litellm_core_utils/request_timeout_resolver.py

@@ -12,10 +12,15 @@ tell "user asked for this" from "nobody set it". This resolver answers that:
 
 from __future__ import annotations
 
-from typing import Final
+from collections.abc import Mapping
+from typing import Final, Literal, Protocol
+
+from pydantic import TypeAdapter
 
 from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS
 
+_TIMEOUTS_BY_SERVICE_TIER: Final[TypeAdapter[Mapping[str, float] | None]] = TypeAdapter(Mapping[str, float] | None)
+
 
 def get_configured_request_timeout() -> float | None:
     """Return the explicitly-configured ``litellm.request_timeout``, else ``None``."""
@@ -27,3 +32,19 @@ def get_configured_request_timeout() -> float | None:
     if timeout != float(DEFAULT_REQUEST_TIMEOUT_SECONDS):
         return timeout
     return None
+
+
+class _DeploymentParams(Protocol):
+    def get(self, key: str, /) -> object: ...
+
+
+def get_service_tier_timeout(
+    service_tier: object,
+    litellm_params: _DeploymentParams,
+    key: Literal["timeout_by_service_tier", "stream_timeout_by_service_tier"],
+) -> float | None:
+    """Return the deployment's ``key`` timeout for the request's ``service_tier``, else ``None``."""
+    if not isinstance(service_tier, str):
+        return None
+    timeouts: Final = _TIMEOUTS_BY_SERVICE_TIER.validate_python(litellm_params.get(key))
+    return None if timeouts is None else timeouts.get(service_tier)

litellm/litellm_core_utils/streaming_handler.py

@@ -45,7 +45,7 @@ from litellm.types.utils import (
 )
 from litellm.types.utils import GenericStreamingChunk as GChunk
 
-from ..exceptions import OpenAIError
+from ..exceptions import OpenAIError, Timeout
 from .core_helpers import map_finish_reason, process_response_headers
 from .exception_mapping_utils import exception_type, with_deployment_api_base
 from .llm_response_utils.get_api_base import get_api_base
@@ -88,6 +88,16 @@ def _apply_provider_fields(response: ModelResponseStream, fields: Mapping[str, o
         setattr(response, key, value)
 
 
+def _is_transport_timeout(error: BaseException) -> bool:
+    return isinstance(error, httpx.TimeoutException) or (
+        isinstance(error, Timeout) and isinstance(error.__cause__, httpx.TimeoutException)
+    )
+
+
+def _skips_stream_fallback(status_code: int | None) -> bool:
+    return status_code is not None and 400 <= status_code < 500 and status_code != 429
+
+
 def is_async_iterable(obj: object) -> bool:
     """
     Check if an object is an async iterable (can be used with 'async for').
@@ -2384,6 +2394,8 @@ class CustomStreamWrapper:
 
         429 (rate-limit) is explicitly exempted from the 4xx filter because
         it is transient and the Router should switch to another model group.
+        So is a transport read timeout, which maps to a 408 Timeout, including
+        one an inner Responses stream already mapped for the chat bridge.
 
         An error an inner stream already wrapped (the chat-to-Responses bridge
         consumes a Responses stream) is rebuilt around the provider exception
@@ -2448,9 +2460,9 @@ class CustomStreamWrapper:
         # Raise non-retriable client errors directly (skip fallback).
         # Exception: 429 (rate-limit) IS retriable/transient — allow it
         # through so the Router can switch to a different model group.
-        if mapped_status_code is not None and 400 <= mapped_status_code < 500 and mapped_status_code != 429:
-            raise mapped_exception
-        if original_status_code is not None and 400 <= original_status_code < 500 and original_status_code != 429:
+        if not _is_transport_timeout(e) and (
+            _skips_stream_fallback(mapped_status_code) or _skips_stream_fallback(original_status_code)
+        ):
             raise mapped_exception
 
         raise MidStreamFallbackError(

litellm/litellm_core_utils/token_counter.py

@@ -35,6 +35,10 @@ from litellm.constants import (
     TOKEN_COUNTER_MAX_EXACT_CHARS,
 )
 from litellm.litellm_core_utils.asyncify import asyncify
+from litellm.litellm_core_utils.prompt_templates.chat_template_tools import (
+    ChatTemplateToolSection,
+    chat_template_tool_section,
+)
 from litellm.litellm_core_utils.tokenizer import HuggingFaceTokenizer, OpenAIEncoding
 
 if TYPE_CHECKING:
@@ -347,6 +351,9 @@ EXTRAPOLATION_SAMPLES: Final = 16
 T_ParamSpec: Final = ParamSpec("T_ParamSpec")
 T_Retval = TypeVar("T_Retval")
 _COUNT_OFFLOAD_LIMITER: Final = anyio.lowlevel.RunVar[anyio.CapacityLimiter]("litellm_count_offload_limiter")
+_ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE: Final = frozenset(
+    ("name", "input_schema", "description", "cache_control", "strict", "type", "eager_input_streaming")
+)
 
 
 def _count_offload_limiter_for_this_loop() -> anyio.CapacityLimiter:
@@ -404,6 +411,7 @@ class _MessageCountParams:
             self.tokens_per_message = 3
             self.tokens_per_name = 1
         self.count_function = _get_count_function(model, custom_tokenizer)
+        self.tool_section = chat_template_tool_section(model)
 
 
 def token_counter(
@@ -467,7 +475,9 @@ def token_counter(
         num_tokens = _count_messages(params, new_messages, use_default_image_token_count, default_token_count)
         if count_response_tokens is False:
             includes_system_message: Final = any(message.get("role", None) == "system" for message in new_messages)
-            num_tokens += _count_extra(params.count_function, tools, tool_choice, includes_system_message)
+            num_tokens += _count_extra(
+                params.count_function, tools, tool_choice, includes_system_message, params.tool_section
+            )
 
     else:
         raise ValueError("Either text or messages must be provided")
@@ -501,7 +511,8 @@ def messages_reach_token_count(
         _count_messages(params, [message], use_default_image_token_count, None) for message in new_messages
     )
     running_totals: Final = accumulate(
-        per_message_counts, initial=_count_extra(params.count_function, tools, None, includes_system_message)
+        per_message_counts,
+        initial=_count_extra(params.count_function, tools, None, includes_system_message, params.tool_section),
     )
     return any(total >= threshold for total in running_totals)
 
@@ -592,6 +603,7 @@ def _count_extra(
     tools: list[ChatCompletionToolParam] | None,
     tool_choice: ChatCompletionNamedToolChoiceParam | None,
     includes_system_message: bool,
+    tool_section: ChatTemplateToolSection | None,
 ) -> int:
     """Count extra tokens for function definitions and tool choices.
     Args:
@@ -599,26 +611,56 @@ def _count_extra(
         tools (Optional[List[ChatCompletionToolParam]]): The available tools.
         tool_choice (Optional[ChatCompletionNamedToolChoiceParam]): The tool choice.
         includes_system_message (bool): Whether the messages include a system message.
+        tool_section (Optional[ChatTemplateToolSection]): How the model's chat template renders tools, if known.
     """
+    reply_primer_tokens: Final = 3
+    tool_choice_tokens: Final = 0 if tool_section is not None else _count_tool_choice(count_function, tool_choice)
+    return (
+        reply_primer_tokens
+        + _count_tool_definitions(count_function, tools, includes_system_message, tool_section)
+        + tool_choice_tokens
+    )
 
-    num_tokens = 3  # every reply is primed with <|start|>assistant<|message|>
 
-    if tools:
-        num_tokens += count_function(_format_function_definitions(tools))
-        num_tokens += 9  # Additional tokens for function definition of tools
-    # If there's a system message and tools are present, subtract four tokens
-    if tools and includes_system_message:
-        num_tokens -= 4
-    # If tool_choice is 'none', add one token.
-    # If it's an object, add 4 + the number of tokens in the function name.
-    # If it's undefined or 'auto', don't add anything.
+def _count_tool_definitions(
+    count_function: TokenCounterFunction,
+    tools: Sequence[object] | None,
+    includes_system_message: bool,
+    tool_section: ChatTemplateToolSection | None,
+) -> int:
+    if not tools:
+        return 0
+    if tool_section is not None:
+        return _count_chat_template_tools(count_function, tools, includes_system_message, tool_section)
+    system_message_discount: Final = 4 if includes_system_message else 0
+    return count_function(_format_function_definitions(tools)) + 9 - system_message_discount
+
+
+def _count_tool_choice(
+    count_function: TokenCounterFunction,
+    tool_choice: ChatCompletionNamedToolChoiceParam | None,
+) -> int:
     if tool_choice == "none":
-        num_tokens += 1
-    elif isinstance(tool_choice, dict):
-        num_tokens += 7
-        num_tokens += count_function(str(tool_choice["function"]["name"]))
+        return 1
+    if isinstance(tool_choice, dict):
+        return 7 + count_function(str(tool_choice["function"]["name"]))
+    return 0
 
-    return num_tokens
+
+def _count_chat_template_tools(
+    count_function: TokenCounterFunction,
+    tools: Sequence[object],
+    includes_system_message: bool,
+    tool_section: ChatTemplateToolSection,
+) -> int:
+    functions: Final = tuple(
+        function for function in _function_definitions(_tools_sent_as_functions(tools)) if function.get("name")
+    )
+    if not functions:
+        return 0
+    system_turn_tokens: Final = 0 if includes_system_message else tool_section.implicit_system_turn_tokens
+    schemas: Final = tool_section.render_tool_schemas(functions)
+    return tool_section.preamble_tokens + system_turn_tokens + count_function(schemas)
 
 
 def _get_extrapolating_count_function(
@@ -1034,15 +1076,31 @@ def _format_function_definitions(tools: Sequence[object]) -> str:
     lines: Final = []
     lines.append("namespace functions {")
     lines.append("")
-    for tool in tools:
-        if not isinstance(tool, Mapping):
-            continue
-        for function in _function_definitions_for_tool(cast(Mapping[str, object], tool)):
-            lines.extend(_format_single_function_definition(function))
+    for function in _function_definitions(tools):
+        lines.extend(_format_single_function_definition(function))
     lines.append("} // namespace functions")
     return "\n".join(lines)
 
 
+def _tools_sent_as_functions(tools: Sequence[object]) -> tuple[Mapping[str, object], ...]:
+    return tuple(tool for tool in _tool_mappings(tools) if not _is_anthropic_web_search_tool(tool))
+
+
+def _is_anthropic_web_search_tool(tool: Mapping[str, object]) -> bool:
+    return tool.get("name") == "web_search" and "input_schema" not in tool and tool.get("type") != "function"
+
+
+def _tool_mappings(tools: Sequence[object]) -> Iterable[Mapping[str, object]]:
+    for tool in tools:
+        if isinstance(tool, Mapping):
+            yield cast(Mapping[str, object], tool)
+
+
+def _function_definitions(tools: Sequence[object]) -> Iterable[Mapping[str, object]]:
+    for tool in _tool_mappings(tools):
+        yield from _function_definitions_for_tool(tool)
+
+
 def _function_definitions_for_tool(tool: Mapping[str, object]) -> Iterable[Mapping[str, object]]:
     function: Final = tool.get("function")
     if isinstance(function, Mapping):
@@ -1054,12 +1112,21 @@ def _function_definitions_for_tool(tool: Mapping[str, object]) -> Iterable[Mappi
             if isinstance(declaration, Mapping):
                 yield declaration
         return
-    parameters: Final = tool.get("input_schema") or tool.get("parameters") or {}
+    input_schema: Final = tool.get("input_schema")
+    parameters: Final = input_schema or tool.get("parameters") or {}
     normalized_parameters: Final = parameters if isinstance(parameters, Mapping) else {}
+    unmapped_anthropic_keys: Final = (
+        {key: value for key, value in tool.items() if key not in _ANTHROPIC_TOOL_KEYS_MAPPED_BY_BRIDGE}
+        if input_schema is not None
+        else {}
+    )
+    description: Final = tool.get("description")
+    strict: Final = tool.get("strict")
     yield {
         "name": tool.get("name"),
-        "description": tool.get("description"),
-        "parameters": normalized_parameters,
+        "parameters": {**normalized_parameters, **unmapped_anthropic_keys},
+        **({"description": description} if description is not None else {}),
+        **({"strict": bool(strict)} if strict is not None else {}),
     }
 
 

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -30,6 +30,10 @@ from litellm.llms.anthropic.pass_through.adapters.transformation import (
     LiteLLMAnthropicMessagesAdapter,
     is_provider_native_tool_dict,
 )
+from litellm.llms.anthropic.pass_through.convertible_blocks import (
+    keep_non_string_tool_result_texts,
+    scannable_tool_result_request,
+)
 from litellm.llms.anthropic.pass_through.messages.utils import filtered_safeguard_results, redacted_stop_details
 from litellm.llms.anthropic.pass_through.stream_assembly import (
     build_complete_streaming_response,
@@ -493,8 +497,8 @@ class AnthropicMessagesHandler(BaseTranslation):
         ``get_streaming_string_so_far``): raw SSE ``bytes`` -- which may bundle
         several events separated by a blank line -- and an already-parsed event
         ``dict``."""
-        if isinstance(item, dict):
-            return (_as_str_mapping(item),)
+        if (event := as_str_mapping(item)) is not None:
+            return (event,)
         if not isinstance(item, (bytes, bytearray)):
             return ()
         return tuple(
@@ -510,7 +514,7 @@ class AnthropicMessagesHandler(BaseTranslation):
             chat_completion_compatible_request,
             _tool_name_mapping,
         ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
-            anthropic_message_request=cast(AnthropicMessagesRequest, data.copy()),
+            anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)),
             preserve_midturn_system=True,
         )
         return chat_completion_compatible_request
@@ -832,7 +836,7 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     @staticmethod
     def _write_back_structured_messages(
-        data: dict,  # mutable-ok: API message payload
+        data: dict[str, object],  # mutable-ok: API message payload
         structured_messages: list,  # mutable-ok: API message payload
         hoisted_system_message: object = None,
         preserve_system_messages: bool = False,
@@ -899,7 +903,7 @@ class AnthropicMessagesHandler(BaseTranslation):
                 for block in content:
                     if isinstance(block, dict) and block.get("type") == "thinking":
                         block.pop("cache_control", None)
-        data["messages"] = converted
+        data["messages"] = keep_non_string_tool_result_texts(converted, data.get("messages"))
 
     @staticmethod
     def _extract_midturn_system_text(
@@ -1601,8 +1605,8 @@ class AnthropicMessagesHandler(BaseTranslation):
 
     @staticmethod
     def _rewrite_buffered_item(item: object, rewrite_event: _SSEEventRewriter) -> object:
-        if isinstance(item, dict):
-            return _rewritten_event(_as_str_mapping(item), rewrite_event)
+        if (event := as_str_mapping(item)) is not None:
+            return _rewritten_event(event, rewrite_event)
         if isinstance(item, (bytes, bytearray)):
             return AnthropicMessagesHandler._rewrite_sse_events(bytes(item), rewrite_event)
         return item

litellm/llms/anthropic/pass_through/adapters/handler.py

@@ -22,6 +22,7 @@ from litellm.llms.anthropic.pass_through.context_management import (
     PolyfillResult,
     apply_context_management,
 )
+from litellm.llms.anthropic.pass_through.convertible_blocks import reject_malformed_tool_result_text
 from litellm.llms.anthropic.pass_through.safeguards import (
     SafeguardsEvaluator,
     build_safeguards_evaluator,
@@ -731,6 +732,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
 
         proxy_litellm_metadata, user_api_key_auth = _extract_proxy_litellm_metadata(kwargs)
         request_kwargs: Final = _REQUEST_KWARGS.validate_python(kwargs)
+        reject_malformed_tool_result_text(messages, model, request_kwargs)
         sub_call_logging_kwargs: Final = proxy_sub_call_logging_kwargs(request_kwargs)
         sub_call_trace_id: Final = proxy_sub_call_trace_id(request_kwargs)
 
@@ -853,6 +855,8 @@ class LiteLLMMessagesToCompletionTransformationHandler:
                 **kwargs,
             )
 
+        reject_malformed_tool_result_text(messages, model, _REQUEST_KWARGS.validate_python(kwargs))
+
         # Run the context_management polyfill on the sync path too so that
         # ``litellm.messages.create()`` callers don't silently lose edits like
         # ``clear_tool_uses_20250919``. The dispatcher is async (so the

litellm/llms/anthropic/pass_through/adapters/transformation.py

@@ -9,6 +9,7 @@ from pydantic import JsonValue, TypeAdapter
 
 import litellm
 from litellm.litellm_core_utils.hidden_params import get_hidden_params
+from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_part
 from litellm.llms.anthropic.pass_through.utils import (
     is_reasoning_auto_summary_enabled,
     json_copy_or_original,
@@ -1334,7 +1335,8 @@ class LiteLLMAnthropicMessagesAdapter:
         block: Final = cast(Mapping[str, object], item)  # cast-ok: untrusted client payload
         match block.get("type"):
             case "text":
-                return ChatCompletionTextObject(type="text", text=str(block.get("text") or ""))
+                text: Final = tool_result_text_part(block)
+                return None if text is None else ChatCompletionTextObject(type="text", text=text)
             case "image" | "document":
                 return self._tool_result_image_part(block.get("source"))
             case "tool_reference":

litellm/llms/anthropic/pass_through/convertible_blocks.py

@@ -1,11 +1,18 @@
-from collections.abc import Iterator, Mapping, Sequence
-from itertools import chain
+import json
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from functools import partial
+from itertools import chain, groupby
+from operator import itemgetter
 from types import MappingProxyType
-from typing import Final
+from typing import Final, TypeAlias
 
 from pydantic import JsonValue, TypeAdapter, ValidationError
 from typing_extensions import assert_never
 
+import litellm
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
+from litellm.litellm_core_utils.prompt_templates.factory import sanitize_anthropic_tool_use_id
 from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
 from litellm.types.utils import TokenCountMessagesFormat
 
@@ -104,3 +111,245 @@ def unconvertible_blocks_error(
             )
         case _:
             assert_never(messages_format)
+
+
+_OBJECTS: Final = TypeAdapter(tuple[object, ...])
+
+
+def _list_items(value: object) -> tuple[object, ...]:
+    return _OBJECTS.validate_python(value) if isinstance(value, list) else ()
+
+
+@dataclass(frozen=True, slots=True)
+class _KeptPart:
+    block: object
+    scanned_text: str
+
+
+_KeptParts: TypeAlias = Mapping[int, _KeptPart]
+_NO_KEPT_PARTS: Final[_KeptParts] = MappingProxyType({})
+
+
+def _scannable_text(text: object) -> str:
+    return json.dumps(text, default=str, ensure_ascii=False)
+
+
+def _non_str_text(block: object) -> _KeptPart | None:
+    match as_str_mapping(block):
+        case {"type": "text", "text": str()}:
+            return None
+        case {"type": "text", "text": text}:
+            return _KeptPart(block=block, scanned_text=_scannable_text(text))
+        case _:
+            return None
+
+
+def tool_result_text_part(block: object) -> str | None:
+    match block:
+        case {"type": "text", "text": str(text)}:
+            return text
+        case {"type": "text", "text": object()}:
+            return None
+        case {"type": "text"}:
+            return ""
+        case _:
+            return None
+
+
+def tool_result_text_parts(tool_result: Mapping[str, object]) -> tuple[str, ...]:
+    parts: Final = map(tool_result_text_part, _list_items(tool_result.get("content")))
+    return tuple(part for part in parts if part is not None)
+
+
+def _user_content_blocks(message: object) -> tuple[object, ...]:
+    match as_str_mapping(message):
+        case {"role": "user", "content": content}:
+            return _list_items(content)
+        case _:
+            return ()
+
+
+def _with_user_blocks(message: object, transform: Callable[[object], object]) -> object:
+    match as_str_mapping(message):
+        case {"role": "user", "content": list()} as user_message:
+            return {**user_message, "content": [transform(block) for block in _user_content_blocks(user_message)]}
+        case _:
+            return message
+
+
+def _non_string_text_errors(label: str, content: object) -> Iterator[str]:
+    return (
+        f"{label} content.{index}.text: Input should be a valid string"
+        for index, text_block in enumerate(_list_items(content))
+        if _non_str_text(text_block) is not None
+    )
+
+
+def _tool_result_text_errors(block: object) -> Iterator[str]:
+    match block:
+        case {"type": "tool_result", "tool_use_id": object() as tool_use_id, "content": object() as content}:
+            return _non_string_text_errors(f"tool_result {tool_use_id}", content)
+        case {"type": "tool_result", "content": object() as content}:
+            return _non_string_text_errors("tool_result", content)
+        case _:
+            return iter(())
+
+
+def malformed_tool_result_text_error(messages: Iterable[object]) -> str | None:
+    blocks: Final = chain.from_iterable(map(_user_content_blocks, messages))
+    return next(chain.from_iterable(map(_tool_result_text_errors, blocks)), None)
+
+
+def malformed_tool_result_count_error(
+    messages: Iterable[Mapping[str, object]] | None,
+    messages_format: TokenCountMessagesFormat,
+    deployment: Mapping[str, object] | None,
+) -> str | None:
+    if messages_format != "anthropic" or deployment_drops_params(deployment):
+        return None
+    return malformed_tool_result_text_error(messages or ())
+
+
+def _scannable_part(part: object) -> object:
+    match as_str_mapping(part):
+        case {"type": "text", "text": str()}:
+            return part
+        case {"type": "text", "text": text} as text_block:
+            return {**text_block, "text": _scannable_text(text)}
+        case _:
+            return part
+
+
+def _scannable_block(block: object) -> object:
+    match as_str_mapping(block):
+        case {"type": "tool_result", "content": list()} as tool_result:
+            return {
+                **tool_result,
+                "content": [_scannable_part(part) for part in _list_items(tool_result.get("content"))],
+            }
+        case _:
+            return block
+
+
+def _scannable_messages(messages: object) -> object:
+    items: Final = _list_items(messages)
+    return [_with_user_blocks(message, _scannable_block) for message in items] if items else messages
+
+
+def scannable_tool_result_request(request: object) -> object:
+    match as_str_mapping(request):
+        case {"messages": messages} as request_fields:
+            return {**request_fields, "messages": _scannable_messages(messages)}
+        case None:
+            return request
+        case request_fields:
+            return {**request_fields}
+
+
+def _non_string_text_parts(content: object) -> _KeptParts:
+    return MappingProxyType(
+        {index: part for index, block in enumerate(_list_items(content)) if (part := _non_str_text(block)) is not None}
+    )
+
+
+def _tool_result_non_string_texts(block: object) -> tuple[tuple[str, _KeptParts], ...]:
+    match as_str_mapping(block):
+        case {"type": "tool_result", "tool_use_id": str(tool_use_id)} as tool_result:
+            return ((sanitize_anthropic_tool_use_id(tool_use_id), _non_string_text_parts(tool_result.get("content"))),)
+        case _:
+            return ()
+
+
+def _content_blocks(content: object) -> tuple[object, ...]:
+    match content:
+        case str() if content:
+            return ({"type": "text", "text": content},)
+        case _:
+            return _list_items(content)
+
+
+def _restored_part(block: object, kept: _KeptPart | None) -> object:
+    return kept.block if kept is not None and tool_result_text_part(block) == kept.scanned_text else block
+
+
+def _with_kept_parts(block: object, tool_result: Mapping[str, object], parts: _KeptParts) -> object:
+    if not parts:
+        return block
+    blocks: Final = _content_blocks(tool_result.get("content"))
+    return {**tool_result, "content": [_restored_part(part, parts.get(index)) for index, part in enumerate(blocks)]}
+
+
+def _restored_block(block: object, kept: Mapping[str, Iterator[_KeptParts]]) -> object:
+    match as_str_mapping(block):
+        case {"type": "tool_result", "tool_use_id": str(tool_use_id)} as tool_result if (
+            sanitize_anthropic_tool_use_id(tool_use_id) in kept
+        ):
+            return _with_kept_parts(
+                block, tool_result, next(kept[sanitize_anthropic_tool_use_id(tool_use_id)], _NO_KEPT_PARTS)
+            )
+        case _:
+            return block
+
+
+def keep_non_string_tool_result_texts(rewritten: Sequence[object], original: object) -> Sequence[object]:
+    blocks: Final = chain.from_iterable(map(_user_content_blocks, _list_items(original)))
+    by_id: Final = tuple(sorted(chain.from_iterable(map(_tool_result_non_string_texts, blocks)), key=itemgetter(0)))
+    if not any(parts for _, parts in by_id):
+        return rewritten
+    kept_in_order: Final = MappingProxyType(
+        {
+            tool_use_id: iter(tuple(map(itemgetter(1), group)))
+            for tool_use_id, group in groupby(by_id, key=itemgetter(0))
+        }
+    )
+    return [_with_user_blocks(message, partial(_restored_block, kept=kept_in_order)) for message in rewritten]
+
+
+def _without_non_string_text(block: object) -> object:
+    match as_str_mapping(block):
+        case {"type": "tool_result", "content": list()} as tool_result:
+            parts: Final = _list_items(tool_result.get("content"))
+            return {**tool_result, "content": [part for part in parts if _non_str_text(part) is None]}
+        case _:
+            return block
+
+
+def _without_non_string_texts(
+    message: dict[str, object],  # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
+) -> dict[str, object]:  # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
+    match message:
+        case {"role": "user", "content": list()}:
+            return {**message, "content": [_without_non_string_text(block) for block in _user_content_blocks(message)]}
+        case _:
+            return message
+
+
+def drop_non_string_tool_result_texts(
+    messages: Sequence[dict[str, object]],  # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
+    drop_params: object,
+) -> Sequence[dict[str, object]]:  # mutable-ok: AnthropicMessagesRequest.messages is typed list[dict]
+    if not drops_unsupported_params(drop_params) or malformed_tool_result_text_error(messages) is None:
+        return messages
+    return [_without_non_string_texts(message) for message in messages]
+
+
+def deployment_drops_params(deployment: Mapping[str, object] | None) -> bool:
+    match deployment:
+        case {"litellm_params": {"drop_params": object() as drop_params}}:
+            return drops_unsupported_params(drop_params)
+        case _:
+            return drops_unsupported_params(None)
+
+
+def reject_malformed_tool_result_text(
+    messages: Iterable[Mapping[str, object]], model: str, forwarded_kwargs: Mapping[str, object]
+) -> None:
+    if drops_unsupported_params(forwarded_kwargs.get("drop_params")):
+        return
+    error: Final = malformed_tool_result_text_error(messages)
+    if error is None:
+        return
+    provider: Final = forwarded_kwargs.get("custom_llm_provider")
+    raise litellm.BadRequestError(
+        message=error, model=model, llm_provider=provider if isinstance(provider, str) else ""
+    )

litellm/llms/anthropic/pass_through/messages/transformation.py

@@ -8,6 +8,7 @@ from litellm._logging import verbose_logger
 from litellm.anthropic_beta_headers_manager import get_unsupported_headers, is_beta_header_supported_by_bundled_config
 from litellm.exceptions import AuthenticationError
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.anthropic.pass_through.convertible_blocks import drop_non_string_tool_result_texts
 from litellm.llms.anthropic.pass_through.safeguards import proxy_serves_safeguards
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
@@ -692,7 +693,9 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             messages = strip_advisor_blocks_from_messages(messages)
 
         anthropic_messages_request: Final[AnthropicMessagesRequest] = AnthropicMessagesRequest(
-            messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(messages),
+            messages=strip_encrypted_reasoning_blocks_from_anthropic_messages(
+                drop_non_string_tool_result_texts(messages, getattr(litellm_params, "drop_params", None))
+            ),
             max_tokens=max_tokens,
             model=model,
             **anthropic_messages_optional_request_params,

litellm/llms/anthropic/pass_through/responses_adapters/handler.py

@@ -14,7 +14,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
     AnthropicCacheControlHook,
     supports_openai_prompt_cache_breakpoint,
 )
-from litellm.litellm_core_utils.core_helpers import normalize_drop_params
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import cache_control_as_prompt_cache_breakpoint
 from litellm.responses.utils import auto_reasoning_summary
 from litellm.types.integrations.custom_logger import CONVERTED_STREAM_KEYS
@@ -32,6 +32,7 @@ from litellm.types.llms.openai import ResponsesAPIResponse
 from litellm.utils import ProviderConfigManager
 
 from ..adapters.handler import request_safeguards_evaluator
+from ..convertible_blocks import reject_malformed_tool_result_text
 from ..safeguards import with_safeguard_results
 from ..utils import litellm_logging_obj_from_kwargs, local_model_name
 from .streaming_iterator import AnthropicResponsesStreamWrapper
@@ -96,7 +97,7 @@ def _prompt_cache_block_rewrite(model: str, forwarded_kwargs: Mapping[str, objec
         return cache_control_as_prompt_cache_breakpoint
     if _model_takes_prompt_cache_breakpoint(model, forwarded_kwargs):
         return None
-    if normalize_drop_params(forwarded_kwargs.get("drop_params")) is True or litellm.drop_params is True:
+    if drops_unsupported_params(forwarded_kwargs.get("drop_params")):
         return _without_prompt_cache_breakpoint
     return None
 
@@ -142,6 +143,7 @@ def _build_responses_kwargs(
     Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses().
     """
     forwarded_kwargs: Final = _forwarded_kwargs(extra_kwargs)
+    reject_malformed_tool_result_text(messages, model, forwarded_kwargs)
     block_rewrite: Final = _prompt_cache_block_rewrite(model, forwarded_kwargs)
     request_messages: Final[AnthropicRequestMessages] = (
         [

litellm/llms/anthropic/pass_through/responses_adapters/transformation.py

@@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.litellm_core_utils.reasoning_effort_utils import (
     reasoning_effort_from_thinking_budget,
 )
+from litellm.llms.anthropic.pass_through.convertible_blocks import tool_result_text_parts
 from litellm.llms.anthropic.pass_through.messages.utils import (
     refusal_stop_details,
     responses_output_refusal_text,
@@ -371,10 +372,9 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                         elif btype == "tool_result":
                             tool_use_id = block.get("tool_use_id", "")
                             inner = block.get("content")
+                            tool_result_block = cast(Mapping[str, object], block)  # cast-ok: TypedDict upcast
                             tool_breakpoint, covered_image_parts, later_image_parts = (
-                                self._tool_result_images_by_breakpoint(
-                                    cast(Mapping[str, object], block)  # cast-ok: TypedDict upcast
-                                )
+                                self._tool_result_images_by_breakpoint(tool_result_block)
                             )
                             document_candidates = (
                                 tuple(
@@ -391,10 +391,7 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                             elif isinstance(inner, str):
                                 output_text = inner
                             elif isinstance(inner, list):
-                                parts = [
-                                    c.get("text", "") for c in inner if isinstance(c, dict) and c.get("type") == "text"
-                                ]
-                                output_text = "\n".join(parts)
+                                output_text = "\n".join(tool_result_text_parts(tool_result_block))
                                 if covered_image_parts or later_image_parts:
                                     output_text = (
                                         f"{output_text}\n{TOOL_RESULT_IMAGE_PLACEHOLDER}"

litellm/llms/azure/chat/gpt_5_transformation.py

@@ -1,12 +1,10 @@
 """Support for Azure OpenAI gpt-5 model family."""
 
-from typing import Final
+from typing import ClassVar, Final
 
 import litellm
-from litellm.exceptions import UnsupportedParamsError
 from litellm.llms.openai.chat.gpt_5_transformation import (
     OpenAIGPT5Config,
-    get_effort_level,
     is_gpt_reasoning_series_name,
 )
 from litellm.types.llms.openai import AllMessageValues
@@ -15,9 +13,16 @@ from .gpt_transformation import AzureOpenAIConfig
 
 
 class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
-    """Azure specific handling for gpt-5 models."""
+    """Azure specific handling for gpt-5 models.
+
+    reasoning_effort none joins xhigh as opt-in here: Azure rejected it on every gpt-5
+    deployment before gpt-5.1 (https://github.com/BerriAI/litellm/issues/16704), and a deployment
+    name the map never heard of must not forward it. See
+    https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning
+    """
 
     GPT5_SERIES_ROUTE = "gpt5_series/"
+    OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh", "none"))
 
     @classmethod
     def _model_map_lookup_name(cls, model: str) -> str:
@@ -27,11 +32,18 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
         model_prices_and_context_window.json; ``azure/gpt-5.1`` is. Overriding the shared
         resolver rather than one lookup means the supports, explicitly-disabled and
         default-effort answers all read the same entry.
+
+        An ``azure_ai/`` name reaches this config when ``get_llm_provider`` re-routes a Foundry
+        deployment on an OpenAI-v1 host to the azure provider with the prefix intact. It is the
+        same product the azure_ai config reads ``azure_ai/<model>`` for, so that row answers when
+        the map carries it, and the azure/ twin when it does not.
         """
         if model.startswith(cls.GPT5_SERIES_ROUTE):
             return "azure/" + model[len(cls.GPT5_SERIES_ROUTE) :]
         if model.startswith("azure/"):
             return model
+        if model.startswith("azure_ai/"):
+            return model if model in litellm.model_cost else "azure/" + model.removeprefix("azure_ai/")
         return "azure/" + model
 
     @classmethod
@@ -75,33 +87,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
         drop_params: bool,
         api_version: str = "",
     ) -> dict:
-        reasoning_effort_value = non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort")
-        effective_effort: Final = get_effort_level(reasoning_effort_value)
-
-        # gpt-5.1/5.2/5.4 support reasoning_effort='none', but other gpt-5 models don't
-        # See: https://learn.microsoft.com/en-us/azure/ai-foundry/openai/how-to/reasoning
-        supports_none: Final = self._supports_reasoning_effort_level(model, "none")
-
-        if effective_effort == "none" and not supports_none:
-            if litellm.drop_params is True or (drop_params is not None and drop_params is True):
-                non_default_params = non_default_params.copy()
-                optional_params = optional_params.copy()
-                if get_effort_level(non_default_params.get("reasoning_effort")) == "none":
-                    non_default_params.pop("reasoning_effort")
-                if get_effort_level(optional_params.get("reasoning_effort")) == "none":
-                    optional_params.pop("reasoning_effort")
-            else:
-                raise UnsupportedParamsError(
-                    status_code=400,
-                    message=(
-                        "Azure OpenAI does not support reasoning_effort='none' for this model. "
-                        "To drop this parameter, set `litellm.drop_params=True` or for proxy:\n\n"
-                        "`litellm_settings:\n drop_params: true`\n"
-                        "Issue: https://github.com/BerriAI/litellm/issues/16704"
-                    ),
-                )
-
-        result: Final = OpenAIGPT5Config.map_openai_params(
+        return OpenAIGPT5Config.map_openai_params(
             self,
             non_default_params=non_default_params,
             optional_params=optional_params,
@@ -109,17 +95,6 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
             drop_params=drop_params,
         )
 
-        # Only drop reasoning_effort='none' for models that don't support it
-        result_effort: Final = get_effort_level(result.get("reasoning_effort"))
-        if result_effort == "none" and not supports_none:
-            result.pop("reasoning_effort")
-
-        # Azure gpt-5.4+ with tools + reasoning_effort is now routed to the
-        # Responses API bridge (same as OpenAI), so we no longer need to drop
-        # reasoning_effort here.  See: responses_api_bridge_check() in main.py.
-
-        return result
-
     def transform_request(
         self,
         model: str,

litellm/llms/azure/common_utils.py

@@ -3,13 +3,15 @@ import hashlib
 import json
 import os
 import re
-from collections.abc import Callable, Mapping
+from collections.abc import Awaitable, Callable, Mapping
+from contextlib import closing
 from functools import lru_cache
 from types import MappingProxyType
 from typing import Final, Literal, NamedTuple, cast
 
 import httpx
 from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
+from pydantic import ConfigDict, TypeAdapter, with_config
 from typing_extensions import ReadOnly, TypedDict
 
 import litellm
@@ -17,6 +19,7 @@ from litellm._logging import verbose_logger
 from litellm.caching.caching import DualCache
 from litellm.constants import DEFAULT_MAX_RETRIES
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.custom_httpx.http_handler import HTTPHandler
 from litellm.llms.openai.common_utils import BaseOpenAILLM, OpenAIAsyncHTTPClient, OpenAIHTTPClient
 from litellm.secret_managers.get_azure_ad_token_provider import (
     get_azure_ad_token_provider,
@@ -38,6 +41,16 @@ class _AzureV1ClientParams(TypedDict, total=False, extra_items=object):
     base_url: ReadOnly[str]
 
 
+@with_config(ConfigDict(extra="allow"))
+class _AsyncAzureClientParams(TypedDict, total=False, extra_items=object):
+    api_key: ReadOnly[str | None]
+    azure_ad_token: ReadOnly[str | None]
+    azure_ad_token_provider: ReadOnly[Callable[[], str | Awaitable[str]] | None]
+
+
+_ASYNC_AZURE_CLIENT_PARAMS_ADAPTER: Final = TypeAdapter(_AsyncAzureClientParams)
+
+
 class _AzureGatewayClientParams(TypedDict, total=False, extra_items=object):
     api_version: ReadOnly[str]
     base_url: ReadOnly[str]
@@ -98,11 +111,42 @@ def _cached_entra_id_token_provider(
     return get_bearer_token_provider(ClientSecretCredential(tenant_id, client_id, client_secret), scope)
 
 
+@lru_cache(maxsize=128)
+def _cached_username_password_token_provider(
+    client_id: str,
+    azure_username: str,
+    azure_password: str,
+    scope: str,
+) -> Callable[[], str]:
+    from azure.identity import UsernamePasswordCredential, get_bearer_token_provider
+
+    credential: Final = UsernamePasswordCredential(
+        client_id=client_id,
+        username=azure_username,
+        password=azure_password,
+    )
+    return get_bearer_token_provider(credential, scope)
+
+
 @lru_cache(maxsize=128)
 def _cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]:
     return get_azure_ad_token_provider(azure_scope=scope)
 
 
+def cached_azure_ad_token_refresh_provider(scope: str) -> Callable[[], str]:
+    return _cached_azure_ad_token_refresh_provider(scope)
+
+
+def _async_azure_ad_token_provider(
+    provider: Callable[[], str | Awaitable[str]],
+) -> Callable[[], Awaitable[str]]:
+    async def async_provider() -> str:
+        token: Final = await asyncio.to_thread(provider)
+        return token if isinstance(token, str) else await token
+
+    return async_provider
+
+
 def get_azure_ad_token_from_entra_id(
     tenant_id: str,
     client_id: str,
@@ -177,22 +221,19 @@ def get_azure_ad_token_from_username_password(
     Returns:
         callable that returns a bearer token.
     """
-    from azure.identity import UsernamePasswordCredential, get_bearer_token_provider
-
     verbose_logger.debug(
         "client_id=%s, azure_username=[set=%s], azure_password=[set=%s]",
         client_id,
         azure_username is not None,
         azure_password is not None,
     )
-    credential: Final = UsernamePasswordCredential(
+    token_provider: Final = _cached_username_password_token_provider(
         client_id=client_id,
-        username=azure_username,
-        password=azure_password,
+        azure_username=azure_username,
+        azure_password=azure_password,
+        scope=scope,
     )
 
-    token_provider: Final = get_bearer_token_provider(credential, scope)
-
     verbose_logger.debug("token_provider %s", token_provider)
 
     return token_provider
@@ -248,18 +289,17 @@ def get_azure_ad_token_from_oidc(
     if azure_ad_token_access_token is not None:
         return azure_ad_token_access_token
 
-    client: Final = litellm.module_level_client
-
-    req_token: Final = client.post(
-        f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token",
-        data={
-            "client_id": azure_client_id,
-            "grant_type": "client_credentials",
-            "scope": scope,
-            "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
-            "client_assertion": oidc_token,
-        },
-    )
+    with closing(HTTPHandler(timeout=litellm.request_timeout)) as client:
+        req_token: Final = client.post(
+            f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token",
+            data={
+                "client_id": azure_client_id,
+                "grant_type": "client_credentials",
+                "scope": scope,
+                "client_assertion_type": "urn:ietf:params:oauth:client-assertion-type:jwt-bearer",
+                "client_assertion": oidc_token,
+            },
+        )
 
     if req_token.status_code != 200:
         raise AzureOpenAIError(
@@ -375,7 +415,7 @@ def get_azure_ad_token(
             "Using Azure AD token provider based on Service Principal with Secret workflow or DefaultAzureCredential for Azure Auth"
         )
         try:
-            azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope)
+            azure_ad_token_provider = _cached_azure_ad_token_refresh_provider(scope)
         except ValueError:
             verbose_logger.debug("Azure AD Token Provider could not be used.")
         except Exception as e:
@@ -560,22 +600,19 @@ class BaseAzureLLM(BaseOpenAILLM):
             # on every request (via `_refresh_api_key`), so passing
             # `azure_ad_token_provider` directly preserves Azure AD token refresh
             # behavior that the regular AzureOpenAI client provides.
-            v1_api_key: str | Callable[[], object] | None = (
-                azure_client_params.get("api_key")
-                or azure_client_params.get("azure_ad_token_provider")
-                or azure_client_params.get("azure_ad_token")
+            v1_client_params: Final[_AsyncAzureClientParams] = _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER.validate_python(
+                azure_client_params
+            )
+            v1_token_provider: Final = (
+                v1_client_params.get("api_key")
+                or v1_client_params.get("azure_ad_token_provider")
+                or v1_client_params.get("azure_ad_token")
+            )
+            v1_api_key: Final[str | Callable[[], str | Awaitable[str]] | None] = (
+                _async_azure_ad_token_provider(v1_token_provider)
+                if _is_async is True and callable(v1_token_provider)
+                else v1_token_provider
             )
-            if _is_async is True and callable(v1_api_key):
-                # AsyncOpenAI expects an async provider; wrap the sync provider
-                # returned by azure-identity. Offload to a thread so a token
-                # refresh (blocking HTTP call to AAD on cache miss) does not
-                # stall the event loop.
-                _sync_provider: Final = v1_api_key
-
-                async def _async_v1_api_key() -> str:
-                    return await asyncio.to_thread(_sync_provider)
-
-                v1_api_key = _async_v1_api_key
 
             v1_params: Final[_AzureV1ClientParams] = {
                 "api_key": v1_api_key,
@@ -597,7 +634,19 @@ class BaseAzureLLM(BaseOpenAILLM):
         else:
             # Traditional Azure API uses AzureOpenAI client
             if _is_async is True:
-                openai_client = AsyncAzureOpenAI(**azure_client_params)
+                async_azure_client_params: Final[_AsyncAzureClientParams] = (
+                    _ASYNC_AZURE_CLIENT_PARAMS_ADAPTER.validate_python(azure_client_params)
+                )
+                async_token_provider_candidate: Final = async_azure_client_params.get("azure_ad_token_provider")
+                async_azure_client_params_with_threaded_provider: Final[_AsyncAzureClientParams] = (
+                    {
+                        **async_azure_client_params,
+                        "azure_ad_token_provider": _async_azure_ad_token_provider(async_token_provider_candidate),
+                    }
+                    if async_token_provider_candidate is not None
+                    else async_azure_client_params
+                )
+                openai_client = AsyncAzureOpenAI(**async_azure_client_params_with_threaded_provider)
             else:
                 openai_client = AzureOpenAI(**azure_client_params)
 

litellm/llms/azure/realtime/http_transformation.py

@@ -7,17 +7,21 @@ from litellm.llms.base_llm.realtime.http_transformation import BaseRealtimeHTTPC
 from litellm.secret_managers.main import get_secret_str
 
 
+def _azure_realtime_v1_url(api_base: str, endpoint: str) -> str:
+    """Azure serves WebRTC only on the GA v1 surface, which rejects any api-version query param."""
+    base: Final = api_base.rstrip("/").removesuffix("/v1").removesuffix("/openai")
+    return f"{base}/openai/v1/realtime/{endpoint}"
+
+
 class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
-    def get_api_base(self, api_base: str | None, **kwargs) -> str:
+    def get_api_base(self, api_base: str | None, **kwargs: object) -> str:
         return api_base or litellm.api_base or get_secret_str("AZURE_API_BASE") or ""
 
-    def get_api_key(self, api_key: str | None, **kwargs) -> str:
+    def get_api_key(self, api_key: str | None, **kwargs: object) -> str:
         return api_key or litellm.api_key or get_secret_str("AZURE_API_KEY") or ""
 
     def get_complete_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
-        base: Final = self.get_api_base(api_base).rstrip("/")
-        version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
-        return f"{base}/openai/realtime/client_secrets?api-version={version}"
+        return _azure_realtime_v1_url(self.get_api_base(api_base), "client_secrets")
 
     def validate_environment(
         self,
@@ -32,16 +36,14 @@ class AzureRealtimeHTTPConfig(BaseRealtimeHTTPConfig):
         }
 
     def get_realtime_calls_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
-        base: Final = self.get_api_base(api_base).rstrip("/")
-        version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
-        return f"{base}/openai/realtime/calls?api-version={version}"
+        return _azure_realtime_v1_url(self.get_api_base(api_base), "calls")
 
     def get_transcription_session_url(self, api_base: str | None, model: str, api_version: str | None = None) -> str:
         base: Final = self.get_api_base(api_base).rstrip("/")
         version: Final = api_version or get_secret_str("AZURE_API_VERSION") or "2024-12-17"
         return f"{base}/openai/realtime/transcription_sessions?api-version={version}"
 
-    def get_realtime_calls_headers(self, ephemeral_key: str) -> dict:
+    def get_realtime_calls_headers(self, ephemeral_key: str) -> dict[str, str]:
         return {
-            "api-key": ephemeral_key,
+            "Authorization": f"Bearer {ephemeral_key}",
         }

litellm/llms/azure/responses/transformation.py

@@ -65,10 +65,18 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def custom_llm_provider(self) -> LlmProviders:
         return LlmProviders.AZURE
 
+    @staticmethod
+    def _is_gpt_5_model(model: str) -> bool:
+        return AzureOpenAIGPT5Config.is_model_gpt_5_model(model)
+
     @staticmethod
     def _supports_reasoning_effort_none(model: str) -> bool:
         return AzureOpenAIGPT5Config.supports_reasoning_effort_level(model, "none")
 
+    @staticmethod
+    def _effort_level_is_disabled(model: str, level: str) -> bool:
+        return AzureOpenAIGPT5Config.row_disables_reasoning_effort_level(model, level)
+
     @staticmethod
     def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
         return AzureOpenAIGPT5Config.effort_resolves_to_none(model, effort)

litellm/llms/azure/search/transformation.py

@@ -35,6 +35,7 @@ from typing import TYPE_CHECKING, Final, Literal
 import httpx
 from pydantic import ConfigDict, ValidationError
 
+from litellm.llms.azure.common_utils import cached_azure_ad_token_refresh_provider
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.search.transformation import (
     BaseSearchConfig,
@@ -263,9 +264,7 @@ def _search_tool(optional_params: Mapping[str, object]) -> _BingGroundingTool |
 
 
 def _default_entra_token_minter() -> str:
-    from litellm.secret_managers.get_azure_ad_token_provider import get_azure_ad_token_provider
-
-    return get_azure_ad_token_provider(azure_scope=ENTRA_SCOPE)()
+    return cached_azure_ad_token_refresh_provider(ENTRA_SCOPE)()
 
 
 class BingGroundingSearchConfig(BaseSearchConfig):

litellm/llms/azure_ai/responses/transformation.py

@@ -3,6 +3,7 @@ from typing import Final
 import httpx
 
 from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
+from litellm.llms.azure_ai.chat.transformation import AzureAIGPT5Config
 from litellm.llms.azure_ai.common_utils import (
     AzureFoundryModelInfo,
     api_key_header_for_base,
@@ -27,6 +28,18 @@ class AzureAIResponsesAPIConfig(AzureOpenAIResponsesAPIConfig):
     def custom_llm_provider(self) -> LlmProviders:
         return LlmProviders.AZURE_AI
 
+    @staticmethod
+    def _supports_reasoning_effort_none(model: str) -> bool:
+        return AzureAIGPT5Config.supports_reasoning_effort_level(model, "none")
+
+    @staticmethod
+    def _effort_level_is_disabled(model: str, level: str) -> bool:
+        return AzureAIGPT5Config.row_disables_reasoning_effort_level(model, level)
+
+    @staticmethod
+    def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
+        return AzureAIGPT5Config.effort_resolves_to_none(model, effort)
+
     def validate_environment(self, headers: dict, model: str, litellm_params: GenericLiteLLMParams | None) -> dict:
         params: Final = litellm_params or GenericLiteLLMParams()
         auth_headers: Final = get_azure_ai_auth_headers(

litellm/llms/baseten/chat.py

@@ -1,6 +1,10 @@
 from typing import Final
 
+import litellm
 from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
+from litellm.secret_managers.main import get_secret_str
+
+BASETEN_MODEL_API_BASE: Final = "https://inference.baseten.co/v1"
 
 
 class BasetenConfig(OpenAIGPTConfig):
@@ -89,11 +93,39 @@ class BasetenConfig(OpenAIGPTConfig):
         Get the OpenAI compatible provider info for Baseten
         """
         # Default to Model API
-        default_api_base: Final = "https://inference.baseten.co/v1"
+        default_api_base: Final = BASETEN_MODEL_API_BASE
         default_api_key: Final = api_key or "BASETEN_API_KEY"
 
         return default_api_base, default_api_key
 
+    @staticmethod
+    def get_api_key(api_key: str | None = None) -> str | None:
+        return api_key or get_secret_str("BASETEN_API_KEY")
+
+    @staticmethod
+    def get_api_base(api_base: str | None = None) -> str | None:
+        return api_base or get_secret_str("BASETEN_API_BASE") or BASETEN_MODEL_API_BASE
+
+    def get_models(self, api_key: str | None = None, api_base: str | None = None) -> list[str]:
+        """
+        Calls Baseten's Model API `/v1/models` endpoint and returns the live catalog.
+        """
+        resolved_api_key: Final = self.get_api_key(api_key)
+        if resolved_api_key is None:
+            raise ValueError(
+                "BASETEN_API_KEY is not set. Please set the environment variable, to query Baseten's `/models` endpoint."
+            )
+        resolved_api_base: Final = (self.get_api_base(api_base) or BASETEN_MODEL_API_BASE).rstrip("/")
+        response: Final = litellm.module_level_client.get(  # pyright: ignore[reportUnknownMemberType]  # untyped params
+            url=f"{resolved_api_base.removesuffix('/v1')}/v1/models",
+            headers={"Authorization": f"Bearer {resolved_api_key}"},
+        )
+        if response.status_code != 200:
+            raise ValueError(
+                f"Failed to fetch models from Baseten. Status code: {response.status_code}, Response: {response.text}"
+            )
+        return ["baseten/" + model["id"] for model in response.json()["data"]]
+
     @staticmethod
     def is_dedicated_deployment(model: str) -> bool:
         """
@@ -118,4 +150,4 @@ class BasetenConfig(OpenAIGPTConfig):
             return f"https://model-{model_id}.api.baseten.co/environments/production/sync/v1"
         else:
             # Use Model API
-            return "https://inference.baseten.co/v1"
+            return BASETEN_MODEL_API_BASE

litellm/llms/custom_httpx/llm_http_handler.py

@@ -2953,14 +2953,15 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers = responses_api_provider_config.validate_environment(
-            headers=response_api_optional_request_params.get("extra_headers", {}) or {},
-            model=model,
-            litellm_params=litellm_params,
-        )
-
-        if extra_headers:
-            headers.update(extra_headers)
+        validated_headers: Final = {
+            **await asyncio.to_thread(
+                responses_api_provider_config.validate_environment,
+                headers=response_api_optional_request_params.get("extra_headers", {}) or {},
+                model=model,
+                litellm_params=litellm_params,
+            ),
+            **(extra_headers or {}),
+        }
 
         # Check if streaming is requested
         stream = response_api_optional_request_params.get("stream", False)
@@ -2976,7 +2977,7 @@ class BaseLLMHTTPHandler:
             input=input,
             response_api_optional_request_params=response_api_optional_request_params,
             litellm_params=litellm_params,
-            headers=headers,
+            headers=validated_headers,
         )
         data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)
 
@@ -3011,7 +3012,7 @@ class BaseLLMHTTPHandler:
         headers, signed_body = await sign_request_off_loop_if_aws(
             responses_api_provider_config,
             responses_api_provider_config.sign_request,
-            headers=headers,
+            headers=validated_headers,
             optional_params=dict(litellm_params),
             request_data=data,
             api_base=api_base,
@@ -3145,8 +3146,11 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = responses_api_provider_config.validate_environment(
-            headers=extra_headers or {}, model="None", litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            responses_api_provider_config.validate_environment,
+            headers=extra_headers or {},
+            model="None",
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -3399,8 +3403,11 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = responses_api_provider_config.validate_environment(
-            headers=extra_headers or {}, model="None", litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            responses_api_provider_config.validate_environment,
+            headers=extra_headers or {},
+            model="None",
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -3563,8 +3570,11 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = responses_api_provider_config.validate_environment(
-            headers=extra_headers or {}, model="None", litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            responses_api_provider_config.validate_environment,
+            headers=extra_headers or {},
+            model="None",
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -4620,8 +4630,11 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers: Final = responses_api_provider_config.validate_environment(
-            headers=extra_headers or {}, model="None", litellm_params=litellm_params
+        headers: Final = await asyncio.to_thread(
+            responses_api_provider_config.validate_environment,
+            headers=extra_headers or {},
+            model="None",
+            litellm_params=litellm_params,
         )
 
         if extra_headers:
@@ -4795,12 +4808,15 @@ class BaseLLMHTTPHandler:
         else:
             async_httpx_client = client
 
-        headers = responses_api_provider_config.validate_environment(
-            headers=extra_headers or {}, model=model, litellm_params=litellm_params
-        )
-
-        if extra_headers:
-            headers.update(extra_headers)
+        validated_headers: Final = {
+            **await asyncio.to_thread(
+                responses_api_provider_config.validate_environment,
+                headers=extra_headers or {},
+                model=model,
+                litellm_params=litellm_params,
+            ),
+            **(extra_headers or {}),
+        }
 
         api_base: Final = responses_api_provider_config.get_complete_url(
             api_base=litellm_params.api_base,
@@ -4816,14 +4832,14 @@ class BaseLLMHTTPHandler:
             response_api_optional_request_params=response_api_optional_request_params,
             api_base=api_base,
             litellm_params=litellm_params,
-            headers=headers,
+            headers=validated_headers,
         )
         data = BaseResponsesAPIConfig.normalize_responses_api_request_dict(data)
 
         headers, signed_body = await sign_request_off_loop_if_aws(
             responses_api_provider_config,
             responses_api_provider_config.sign_request,
-            headers=headers,
+            headers=validated_headers,
             optional_params=dict(litellm_params),
             request_data=data,
             api_base=url,
@@ -6911,6 +6927,7 @@ class BaseLLMHTTPHandler:
         custom_llm_provider: str | None = None,
         first_message: str | None = None,
         request_defaults: ResponsesWebSocketRequestDefaults | None = None,
+        turn_admission: Callable[[], Awaitable[None]] | None = None,
         frame_guard: "ResponsesWsFrameGuard | None" = None,
         **kwargs: object,
     ) -> Exception | None:
@@ -6973,12 +6990,12 @@ class BaseLLMHTTPHandler:
                 custom_llm_provider=custom_llm_provider,
                 first_message=first_message,
                 quota_callbacks=_ws_quota_callbacks,
+                turn_admission=turn_admission,
                 guardrail_callbacks=_ws_guardrail_callbacks,  # pyright: ignore[reportUnknownArgumentType]  # duck-typed callbacks from litellm.callbacks
                 frame_guard=frame_guard,
                 **kwargs,
             )
-            await handler.run()
-            return None
+            return await handler.run()
 
         import websockets
         from websockets.asyncio.client import ClientConnection
@@ -6990,7 +7007,8 @@ class BaseLLMHTTPHandler:
                 **kwargs,
             }
         )
-        headers: Final = responses_api_provider_config.validate_environment(
+        headers: Final = await asyncio.to_thread(
+            responses_api_provider_config.validate_environment,
             headers={},
             model=model,
             litellm_params=litellm_params,
@@ -7069,6 +7087,7 @@ class BaseLLMHTTPHandler:
                     authorized_model=model,
                     custom_llm_provider=custom_llm_provider,
                     request_defaults=request_defaults,
+                    turn_admission=turn_admission,
                     frame_guard=frame_guard,
                 )
                 return await streaming.bidirectional_forward()

litellm/llms/deepseek/search/__init__.py

@@ -0,0 +1,7 @@
+"""
+DeepSeek Search API module.
+"""
+
+from litellm.llms.deepseek.search.transformation import DeepSeekSearchConfig
+
+__all__ = ["DeepSeekSearchConfig"]

litellm/llms/deepseek/search/transformation.py

@@ -0,0 +1,172 @@
+"""
+DeepSeek Search API.
+Endpoint: POST https://api.deepseek.com/v1/tools/search
+Docs: https://api-docs.deepseek.com/guides/search/
+"""
+
+from __future__ import annotations
+
+from collections.abc import Mapping, Sequence
+from typing import Final
+
+import httpx
+from pydantic import ConfigDict, TypeAdapter, ValidationError
+
+from litellm.exceptions import BadRequestError
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.llms.base_llm.search.transformation import (
+    BaseSearchConfig,
+    SearchResponse,
+    SearchResult,
+)
+from litellm.secret_managers.main import get_secret_str
+
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_OBJECTS: Final = TypeAdapter(Sequence[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True))
+_STR_LIST: Final = TypeAdapter(list[str])
+
+DEEPSEEK_SEARCH_MAX_COUNT: Final = 50
+DEEPSEEK_SEARCH_MAX_QUERY_CHARS: Final = 128
+_DEEPSEEK_SEARCH_PATH: Final = "/v1/tools/search"
+# DeepSeek chat defaults DEEPSEEK_API_BASE to the /beta prefix; the search route lives at the root.
+_CHAT_ONLY_BASE_SUFFIXES: Final = ("/beta", "/v1")
+
+
+class DeepSeekSearchConfig(BaseSearchConfig):
+    DEEPSEEK_API_BASE = "https://api.deepseek.com"
+
+    @staticmethod
+    def ui_friendly_name() -> str:
+        return "DeepSeek"
+
+    def validate_environment(
+        self,
+        headers: dict[str, str],  # mutable-ok: BaseSearchConfig.validate_environment signature
+        api_key: str | None = None,
+        api_base: str | None = None,
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.validate_environment signature
+    ) -> dict[str, str]:  # mutable-ok: the http handler passes this straight to httpx as headers
+        resolved_key: Final = self.resolve_server_api_key(
+            caller_api_key=api_key,
+            caller_api_base=api_base,
+            key_env_vars=("DEEPSEEK_API_KEY",),
+            base_env_var="DEEPSEEK_API_BASE",
+            default_api_base=self.DEEPSEEK_API_BASE,
+        )
+        if not resolved_key:
+            raise ValueError("DEEPSEEK_API_KEY is not set. Set `DEEPSEEK_API_KEY` environment variable.")
+        return {**headers, "Authorization": f"Bearer {resolved_key}", "Content-Type": "application/json"}
+
+    def get_complete_url(
+        self,
+        api_base: str | None,
+        optional_params: dict[str, object],  # mutable-ok: BaseSearchConfig.get_complete_url signature
+        data: dict[str, object] | list[dict[str, object]] | None = None,  # mutable-ok: base signature
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.get_complete_url signature
+    ) -> str:
+        resolved_base: Final = api_base or get_secret_str("DEEPSEEK_API_BASE") or self.DEEPSEEK_API_BASE
+        clean_base: Final = resolved_base.rstrip("/")
+        if clean_base.endswith(_DEEPSEEK_SEARCH_PATH):
+            return clean_base
+        return f"{_strip_chat_suffix(clean_base)}{_DEEPSEEK_SEARCH_PATH}"
+
+    def transform_search_request(
+        self,
+        query: str | list[str],  # mutable-ok: BaseSearchConfig.transform_search_request signature
+        optional_params: dict[str, object],  # mutable-ok: base signature
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.transform_search_request signature
+    ) -> dict[str, object]:  # mutable-ok: the http handler sends this as the JSON body
+        """
+        Unified params -> DeepSeek body: ``query`` (list joined with spaces, ``search_domain_filter``
+        folded in as ``site:`` operators, the whole string capped at 128 characters by DeepSeek),
+        ``max_results`` -> ``count`` clamped to [1, 50].
+        ``country`` and ``max_tokens_per_page`` have no DeepSeek equivalent and are dropped.
+        Everything outside the unified spec (``need_content``, ``ori_question``, ``localization``)
+        is forwarded as-is.
+        """
+        joined_query: Final = " ".join(query) if isinstance(query, list) else query
+        domains: Final = _domain_filter(optional_params.get("search_domain_filter"))
+        count: Final = _count(optional_params.get("max_results"))
+        unified: Final = self.get_supported_perplexity_optional_params()
+        passthrough: Final = {k: v for k, v in optional_params.items() if k not in unified}
+        return {
+            **passthrough,
+            "query": _checked_query(_with_site_filters(joined_query, domains), domains),
+            **({"count": count} if count is not None else {}),
+        }
+
+    def transform_search_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: LiteLLMLoggingObj,
+        **kwargs: object,  # kwargs-ok: BaseSearchConfig.transform_search_response signature
+    ) -> SearchResponse:
+        response_json: Final = _JSON_OBJECT.validate_python(raw_response.json())
+        raw_results: Final = _JSON_OBJECTS.validate_python(response_json.get("results", ()))
+        return SearchResponse(
+            results=[_to_search_result(item) for item in raw_results],
+            object="search",
+        )
+
+
+def _strip_chat_suffix(base: str) -> str:
+    return next(
+        (base.removesuffix(suffix) for suffix in _CHAT_ONLY_BASE_SUFFIXES if base.endswith(suffix)),
+        base,
+    )
+
+
+def _domain_filter(raw: object) -> tuple[str, ...]:
+    try:
+        return tuple(_STR_LIST.validate_python(raw))
+    except ValidationError:
+        return ()
+
+
+def _with_site_filters(query: str, domains: tuple[str, ...]) -> str:
+    if not domains:
+        return query
+    site_clause: Final = " OR ".join(f"site:{domain}" for domain in domains)
+    return f"{query} {site_clause}"
+
+
+def _checked_query(query: str, domains: tuple[str, ...]) -> str:
+    if len(query) <= DEEPSEEK_SEARCH_MAX_QUERY_CHARS:
+        return query
+    filter_note: Final = (
+        f" ({len(domains)} search_domain_filter entries are sent as site: operators inside the query)"
+        if domains
+        else ""
+    )
+    raise BadRequestError(
+        message=(
+            f"DeepSeek Search accepts a query of at most {DEEPSEEK_SEARCH_MAX_QUERY_CHARS} characters, "
+            f"got {len(query)}{filter_note}. Shorten the query or the domain filter."
+        ),
+        model="deepseek/search",
+        llm_provider="deepseek",
+    )
+
+
+def _count(raw: object) -> int | None:
+    if isinstance(raw, bool) or not isinstance(raw, int):
+        return None
+    return max(1, min(raw, DEEPSEEK_SEARCH_MAX_COUNT))
+
+
+def _to_search_result(item: Mapping[str, object]) -> SearchResult:
+    date: Final = item.get("date")
+    return SearchResult.model_validate(
+        {
+            **item,
+            "title": _as_str(item.get("title")),
+            "url": _as_str(item.get("url")),
+            "snippet": _as_str(item.get("snippet")),
+            "date": date if isinstance(date, str) else None,
+            "last_updated": None,
+        }
+    )
+
+
+def _as_str(value: object) -> str:
+    return value if isinstance(value, str) else ""

litellm/llms/gemini/common_utils.py

@@ -508,6 +508,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
             count_context_error_response,
             gemini_count_contents,
             inference_request_view,
+            malformed_tool_result_text_count_error,
             provider_error_response,
             unconvertible_messages_error,
             vertex_gemini_count_context,
@@ -524,7 +525,9 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
         count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools)
         if isinstance(count_context, VertexGeminiCountContextError):
             return count_context_error_response(count_context, request_model, model_to_use)
-        blocks_error: Final = unconvertible_messages_error(request_view.messages, messages_format)
+        blocks_error: Final = unconvertible_messages_error(
+            request_view.messages, messages_format
+        ) or malformed_tool_result_text_count_error(messages, messages_format, deployment)
         if blocks_error is not None:
             return count_context_error_response(blocks_error, request_model, model_to_use)
 

litellm/llms/mistral/chat/transformation.py

@@ -7,6 +7,7 @@ Docs - https://docs.mistral.ai/api/
 """
 
 from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping
+from itertools import accumulate, count
 from typing import TYPE_CHECKING, Final, Literal, cast, get_type_hints, overload
 
 import httpx
@@ -53,6 +54,9 @@ def _accepted_reasoning_effort(model: str, requested: str, custom_llm_provider:
     return accepted
 
 
+_NO_TOOL_CALL_TURN: Final[tuple[int, frozenset[str]]] = (-1, frozenset())
+
+
 class MistralConfig(OpenAIGPTConfig):
     """
     Reference: https://docs.mistral.ai/api/
@@ -273,14 +277,30 @@ class MistralConfig(OpenAIGPTConfig):
         messages = handle_messages_with_content_list_to_str_conversion(messages)
 
         ## 3. Handle name in message
-        new_messages: Final[list[AllMessageValues]] = []
-        for m in messages:
-            m = MistralConfig._handle_name_in_message(m)
-            m = MistralConfig._handle_tool_call_message(m)
-            if MistralConfig._is_empty_assistant_message(m):
-                continue
-            m = strip_none_values_from_message(m)  # prevents 'extra_forbidden' error
-            new_messages.append(m)
+        prepared_messages: Final = tuple(
+            MistralConfig._handle_tool_call_message(MistralConfig._handle_name_in_message(m)) for m in messages
+        )
+        empty_flags: Final = tuple(map(MistralConfig._is_empty_assistant_message, prepared_messages))
+        tool_call_turns: Final = accumulate(
+            zip(count(), prepared_messages, empty_flags, strict=False),
+            MistralConfig._tool_call_turn,
+            initial=_NO_TOOL_CALL_TURN,
+        )
+        tool_reply_keys: Final = tuple(map(MistralConfig._tool_reply_key, prepared_messages, tool_call_turns))
+        first_reply_positions: Final = {
+            key: position for position, key in reversed(tuple(enumerate(tool_reply_keys))) if key is not None
+        }
+        keys_answered_with_content: Final = frozenset(
+            key for key, is_empty in zip(tool_reply_keys, empty_flags, strict=True) if not is_empty
+        )
+        answering_positions: Final = frozenset(
+            position for key, position in first_reply_positions.items() if key not in keys_answered_with_content
+        )
+        new_messages: Final[list[AllMessageValues]] = [
+            strip_none_values_from_message(MistralConfig._with_tool_reply_content(m))
+            for position, (m, is_empty) in enumerate(zip(prepared_messages, empty_flags, strict=True))
+            if not is_empty or position in answering_positions
+        ]
 
         if is_async:
             return super()._transform_messages(new_messages, model, True)
@@ -469,6 +489,44 @@ class MistralConfig(OpenAIGPTConfig):
             message["tool_calls"] = mistral_tool_calls
         return message
 
+    @staticmethod
+    def _tool_call_id(tool_call: object) -> str | None:
+        match tool_call:
+            case {"id": str(tool_call_id)}:
+                return tool_call_id
+            case _:
+                return None
+
+    @staticmethod
+    def _tool_call_turn(
+        turn: tuple[int, frozenset[str]], message_entry: tuple[int, AllMessageValues, bool]
+    ) -> tuple[int, frozenset[str]]:
+        match message_entry:
+            case (position, {"role": "assistant", "tool_calls": list(tool_calls)}, _):
+                return position, frozenset(filter(None, map(MistralConfig._tool_call_id, tool_calls)))
+            case (_, {"role": "tool"}, _) | (_, _, True):
+                return turn
+            case _:
+                return _NO_TOOL_CALL_TURN
+
+    @staticmethod
+    def _tool_reply_key(message: AllMessageValues, turn: tuple[int, frozenset[str]]) -> tuple[int, str] | None:
+        match message, turn:
+            case {"role": "tool", "tool_call_id": str(tool_call_id)}, (position, tool_call_ids) if (
+                tool_call_id in tool_call_ids
+            ):
+                return position, tool_call_id
+            case _:
+                return None
+
+    @staticmethod
+    def _with_tool_reply_content(message: AllMessageValues) -> AllMessageValues:
+        match message:
+            case {"role": "tool"} if not message.get("content"):
+                return {**message, "content": ""}
+            case _:
+                return message
+
     @classmethod
     def _is_empty_assistant_message(cls, message: AllMessageValues) -> bool:
         """

litellm/llms/openai/chat/gpt_5_transformation.py

@@ -1,7 +1,7 @@
 """Support for OpenAI gpt-5 model family."""
 
 import re
-from typing import Final
+from typing import ClassVar, Final
 
 import litellm
 from litellm.utils import (
@@ -71,6 +71,10 @@ _get_effort_level = get_effort_level
 GPT_REASONING_SERIES_MARKERS: Final = ("gpt-5", "gpt-6")
 
 
+def unsupported_reasoning_effort_message(param: str, level: str, model: str) -> str:
+    return f"{param}={level} is not supported for {model}. To drop unsupported params set `litellm.drop_params = True`"
+
+
 def is_gpt_reasoning_series_name(model: str) -> bool:
     normalized: Final = model.split("/")[-1]
     return any(marker in model for marker in GPT_REASONING_SERIES_MARKERS) and "gpt-5-chat" not in normalized
@@ -86,6 +90,9 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
     - Support for GPT-5-Codex models optimized for code generation.
     """
 
+    OPT_IN_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("xhigh",))
+    ROUTE_SPECIFIC_REASONING_EFFORTS: ClassVar[frozenset[str]] = frozenset(("max",))
+
     @classmethod
     def is_model_gpt_5_model(cls, model: str) -> bool:
         return is_gpt_reasoning_series_name(model)
@@ -205,15 +212,10 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
         return False
 
     @classmethod
-    def _is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool:
+    def is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool:
         """Return True only when the model map explicitly sets the capability to False.
 
-        Unlike ``_supports_reasoning_effort_level`` (which requires an explicit True),
-        this method returns True only when ``supports_{level}_reasoning_effort`` is
-        explicitly set to ``False`` in the model map.  A missing key is treated as
-        supported (i.e. this method returns False = not disabled).
-
-        Use this for opt-out checks where unknown models should be allowed through.
+        A missing key is not a disable. Unknown models pass through.
         """
         return is_explicitly_disabled_factory(
             model=cls._model_map_lookup_name(model),
@@ -221,6 +223,39 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
             key=f"supports_{level}_reasoning_effort",
         )
 
+    @classmethod
+    def _is_reasoning_effort_level_explicitly_disabled(cls, model: str, level: str) -> bool:
+        return cls.is_reasoning_effort_level_explicitly_disabled(model, level)
+
+    @classmethod
+    def refuses_reasoning_effort_level(cls, model: str, level: str) -> bool:
+        """The chat gate for any requested effort level, read off the model's map row alone.
+
+        An opt-in level (xhigh here; Azure adds none) is refused unless the row says true, since
+        the providers rejected it on most of the family when it arrived. Every other level is
+        refused only when the row says false: a model the map never heard of, or a row without the
+        flag, keeps forwarding what it was sent, and a level with no flag at all (medium, high) is
+        never refused. So a new model in the family needs its map row and no code change, the
+        row's supports_<level>_reasoning_effort flags being the whole contract.
+
+        A route-specific level (max) is never refused by either gate: the same model honors it
+        on Responses and refuses it on Chat Completions, so one row flag cannot gate it on
+        either route. Chat forwards it for the Responses bridge and a plain chat call gets the
+        provider's own 400, exactly as before the gate read every level.
+        """
+        if level in cls.ROUTE_SPECIFIC_REASONING_EFFORTS:
+            return False
+        if level in cls.OPT_IN_REASONING_EFFORTS:
+            return not cls._supports_reasoning_effort_level(model, level)
+        return cls.is_reasoning_effort_level_explicitly_disabled(model, level)
+
+    @classmethod
+    def row_disables_reasoning_effort_level(cls, model: str, level: str) -> bool:
+        """The Responses gate: opt-out for every level except the route-specific ones."""
+        if level in cls.ROUTE_SPECIFIC_REASONING_EFFORTS:
+            return False
+        return cls.is_reasoning_effort_level_explicitly_disabled(model, level)
+
     def get_supported_openai_params(self, model: str) -> list:
         if self.is_model_gpt_5_search_model(model):
             return [
@@ -281,12 +316,12 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
                 drop_params=drop_params,
             )
 
-        # Get raw reasoning_effort and effective effort level for all guards.
-        # Use effective_effort (extracted string) for xhigh validation, "none" checks, and
-        # tool/sampling guards — dict inputs like {"effort": "none", "summary": "detailed"}
-        # must be treated as effort="none" to avoid incorrect tool-drop or sampling errors.
-        raw_reasoning_effort = non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort")
-        effective_effort: Final = get_effort_level(raw_reasoning_effort)
+        # Dict inputs like {"effort": "none", "summary": "detailed"} are judged by their effort
+        # string so the gate, the sampling guard, and the temperature guard all see one level.
+        raw_reasoning_effort: Final = non_default_params.get("reasoning_effort") or optional_params.get(
+            "reasoning_effort"
+        )
+        requested_effort: Final = get_effort_level(raw_reasoning_effort)
 
         # Normalize dict reasoning_effort to string for Chat Completions API.
         # Example: {"effort": "high", "summary": "detailed"} -> "high"
@@ -298,31 +333,20 @@ class OpenAIGPT5Config(OpenAIGPTConfig):
                 if "reasoning_effort" in optional_params:
                     optional_params["reasoning_effort"] = normalized
 
-        if effective_effort == "xhigh":
-            # xhigh is an opt-in capability: only allow if model explicitly supports it.
-            if not self._supports_reasoning_effort_level(model, effective_effort):
-                if litellm.drop_params or drop_params:
-                    non_default_params.pop("reasoning_effort", None)
-                    optional_params.pop("reasoning_effort", None)
-                else:
-                    raise litellm.utils.UnsupportedParamsError(
-                        message=(f"reasoning_effort={effective_effort} is not supported for this model."),
-                        status_code=400,
-                    )
-        elif effective_effort in ("minimal", "low"):
-            # minimal/low are opt-out: unknown models pass through; only block when
-            # the model map explicitly sets supports_{level}_reasoning_effort=false.
-            # Example: gpt-5.5-pro only accepts {medium, high, xhigh}, so it sets
-            # supports_low_reasoning_effort=false (and supports_minimal=false).
-            if self._is_reasoning_effort_level_explicitly_disabled(model, effective_effort):
-                if litellm.drop_params or drop_params:
-                    non_default_params.pop("reasoning_effort", None)
-                    optional_params.pop("reasoning_effort", None)
-                else:
-                    raise litellm.utils.UnsupportedParamsError(
-                        message=(f"reasoning_effort={effective_effort} is not supported for this model."),
-                        status_code=400,
-                    )
+        if requested_effort is not None and self.refuses_reasoning_effort_level(model, requested_effort):
+            if litellm.drop_params or drop_params:
+                non_default_params.pop("reasoning_effort", None)
+                optional_params.pop("reasoning_effort", None)
+            else:
+                raise litellm.utils.UnsupportedParamsError(
+                    message=unsupported_reasoning_effort_message("reasoning_effort", requested_effort, model),
+                    status_code=400,
+                )
+
+        # The level the provider will see: None once dropped, so the guards below judge by the model's default effort
+        effective_effort: Final = get_effort_level(
+            non_default_params.get("reasoning_effort") or optional_params.get("reasoning_effort")
+        )
 
         ################################################################
         # max_tokens is not supported for gpt-5 models on OpenAI API

litellm/llms/openai/responses/count_tokens/token_counter.py

@@ -11,7 +11,7 @@ from typing_extensions import assert_never
 
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.asyncify import asyncify
-from litellm.llms.anthropic.pass_through.convertible_blocks import unconvertible_blocks_error
+from litellm.llms.anthropic.pass_through.convertible_blocks import deployment_drops_params, unconvertible_blocks_error
 from litellm.llms.anthropic.pass_through.responses_adapters.transformation import (
     LiteLLMAnthropicToResponsesAPIAdapter,
 )
@@ -110,7 +110,9 @@ class OpenAITokenCounter(BaseTokenCounter):
             return None
 
         malformed_error: Final = (
-            OpenAICountTokensConfig.malformed_anthropic_messages_error(messages)
+            OpenAICountTokensConfig.malformed_anthropic_messages_error(
+                messages, drop_params=deployment_drops_params(deployment)
+            )
             if messages_format == "anthropic"
             else None
         )

litellm/llms/openai/responses/count_tokens/transformation.py

@@ -10,6 +10,7 @@ from typing import Any, Final, Literal, TypeAlias
 from pydantic import JsonValue, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, TypedDict
 
+from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_text_error
 from litellm.llms.anthropic.pass_through.messages.utils import normalize_native_messages
 from litellm.llms.anthropic.pass_through.responses_adapters.transformation import (
     LiteLLMAnthropicToResponsesAPIAdapter,
@@ -295,14 +296,16 @@ class OpenAICountTokensConfig:
         return input_items, instructions
 
     @staticmethod
-    def malformed_anthropic_messages_error(messages: Sequence[Mapping[str, object]]) -> str | None:
+    def malformed_anthropic_messages_error(
+        messages: Sequence[Mapping[str, object]], *, drop_params: bool
+    ) -> str | None:
         try:
             _ANTHROPIC_COUNT_MESSAGES.validate_python(messages)
         except ValidationError as e:
             return "; ".join(
                 f"messages.{'.'.join(str(part) for part in error['loc'])}: {error['msg']}" for error in e.errors()
             )
-        return None
+        return None if drop_params else malformed_tool_result_text_error(messages)
 
     @staticmethod
     def anthropic_messages_to_responses_input(

litellm/llms/openai/responses/transformation.py

@@ -25,7 +25,10 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
-from litellm.llms.openai.chat.gpt_5_transformation import is_gpt_reasoning_series_name
+from litellm.llms.openai.chat.gpt_5_transformation import (
+    is_gpt_reasoning_series_name,
+    unsupported_reasoning_effort_message,
+)
 from litellm.responses.litellm_completion_transformation.custom_tools import TOOL_CALL_ITEM_ID_PREFIX_BY_TYPE
 from litellm.responses.litellm_completion_transformation.reasoning_items import is_litellm_minted_reasoning_item
 from litellm.responses.litellm_completion_transformation.tool_search import (
@@ -278,6 +281,20 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
 
         return supports_none_reasoning_effort(model=model, custom_llm_provider=None)
 
+    @staticmethod
+    def _effort_level_is_disabled(model: str, level: str) -> bool:
+        """Whether the model's cost-map row turns this effort level off.
+
+        The same row the chat gate reads, so a level the row refuses is refused on both wires
+        and a new model needs its row, not code. This surface stays opt-out for every level,
+        xhigh included, since it never refused a level the row omits; max is the exception, a
+        level the row cannot gate because the same model takes it here and refuses it on chat.
+        Azure overrides this so a bare deployment name reads the azure/ entry.
+        """
+        from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
+
+        return OpenAIGPT5Config.row_disables_reasoning_effort_level(model, level)
+
     @staticmethod
     def _effort_resolves_to_none(model: str, effort: str | None) -> bool:
         """Whether this request's reasoning effort ends up as "none", the one condition
@@ -384,7 +401,25 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         lookup_name: Final = self._model_map_lookup_name(model)
         if self._is_gpt_5_model(model=lookup_name):
             reasoning: Final = params.get("reasoning") or {}
-            effort: Final = reasoning.get("effort") if isinstance(reasoning, dict) else None
+            requested_effort: Final = reasoning.get("effort") if isinstance(reasoning, dict) else None
+            if (
+                isinstance(reasoning, dict)
+                and isinstance(requested_effort, str)
+                and self._effort_level_is_disabled(lookup_name, requested_effort)
+            ):
+                if not (drop_params or litellm.drop_params):
+                    raise litellm.UnsupportedParamsError(
+                        message=unsupported_reasoning_effort_message("reasoning.effort", requested_effort, model),
+                        status_code=400,
+                    )
+                remaining: Final = reasoning.copy()
+                remaining.pop("effort", None)
+                if remaining:
+                    params["reasoning"] = remaining
+                else:
+                    params.pop("reasoning", None)
+            forwarded_reasoning: Final = params.get("reasoning")
+            effort: Final = forwarded_reasoning.get("effort") if isinstance(forwarded_reasoning, dict) else None
             supports_none: Final = self._supports_reasoning_effort_none(model=lookup_name)
             effort_is_none: Final = supports_none and self._effort_resolves_to_none(lookup_name, effort)
 

litellm/llms/vertex_ai/common_utils.py

@@ -1375,6 +1375,7 @@ class VertexAITokenCounter(BaseTokenCounter):
                 count_context_error_response,
                 gemini_count_contents,
                 inference_request_view,
+                malformed_tool_result_text_count_error,
                 provider_error_response,
                 unconvertible_messages_error,
                 vertex_gemini_count_context,
@@ -1392,7 +1393,9 @@ class VertexAITokenCounter(BaseTokenCounter):
             count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools)
             if isinstance(count_context, VertexGeminiCountContextError):
                 return count_context_error_response(count_context, request_model, model_to_use)
-            blocks_error: Final = unconvertible_messages_error(request_view.messages, messages_format)
+            blocks_error: Final = unconvertible_messages_error(
+                request_view.messages, messages_format
+            ) or malformed_tool_result_text_count_error(messages, messages_format, deployment)
             if blocks_error is not None:
                 return count_context_error_response(blocks_error, request_model, model_to_use)
 

litellm/llms/vertex_ai/count_tokens/context.py

@@ -132,6 +132,17 @@ def unconvertible_messages_error(
     return None if error_message is None else VertexGeminiCountContextError(error_message=error_message)
 
 
+def malformed_tool_result_text_count_error(
+    messages: Sequence[Mapping[str, object]] | None,
+    messages_format: TokenCountMessagesFormat,
+    deployment: Mapping[str, object] | None,
+) -> VertexGeminiCountContextError | None:
+    from litellm.llms.anthropic.pass_through.convertible_blocks import malformed_tool_result_count_error
+
+    error_message: Final = malformed_tool_result_count_error(messages, messages_format, deployment)
+    return None if error_message is None else VertexGeminiCountContextError(error_message=error_message)
+
+
 def count_context_error_response(
     error: VertexGeminiCountContextError, request_model: str, model_used: str
 ) -> TokenCountResponse:

litellm/model_prices_and_context_window_backup.json

@@ -3840,9 +3840,10 @@
         "supports_vision": true
     },
     "azure/container": {
-        "code_interpreter_cost_per_session": 0.03,
+        "code_interpreter_cost_per_session": 0.033,
         "litellm_provider": "azure",
-        "mode": "chat"
+        "mode": "chat",
+        "source": "https://azure.microsoft.com/en-us/pricing/details/cognitive-services/openai-service/"
     },
     "azure_ai/gpt-oss-120b": {
         "input_cost_per_token": 1.5e-07,
@@ -3894,7 +3895,7 @@
         "supports_max_reasoning_effort": false,
         "supports_minimal_reasoning_effort": false,
         "supports_native_streaming": true,
-        "supports_none_reasoning_effort": true,
+        "supports_none_reasoning_effort": false,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
         "supports_prompt_cache_breakpoint": true,
@@ -8408,7 +8409,7 @@
         "supports_max_reasoning_effort": false,
         "supports_minimal_reasoning_effort": false,
         "supports_native_streaming": true,
-        "supports_none_reasoning_effort": true,
+        "supports_none_reasoning_effort": false,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
         "supports_prompt_caching": true,
@@ -8457,7 +8458,7 @@
         "supports_max_reasoning_effort": false,
         "supports_minimal_reasoning_effort": false,
         "supports_native_streaming": true,
-        "supports_none_reasoning_effort": true,
+        "supports_none_reasoning_effort": false,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
         "supports_prompt_caching": true,
@@ -9251,7 +9252,7 @@
         "supports_max_reasoning_effort": false,
         "supports_minimal_reasoning_effort": false,
         "supports_native_streaming": true,
-        "supports_none_reasoning_effort": true,
+        "supports_none_reasoning_effort": false,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
         "supports_prompt_caching": true,
@@ -24365,6 +24366,11 @@
             "notes": "Grounding with Bing Search (G1 SKU): $14 per 1,000 transactions. Tokens for the Foundry model deployment that runs the grounded search are billed separately on that deployment."
         }
     },
+    "deepseek/search": {
+        "input_cost_per_query": 0.00075,
+        "litellm_provider": "deepseek",
+        "mode": "search"
+    },
     "tinyfish/search": {
         "input_cost_per_query": 0.0,
         "litellm_provider": "tinyfish",
@@ -29726,14 +29732,15 @@
         "google_maps_grounding_cost_per_query": 0.014
     },
     "vertex_ai/gemini-3.8-flash": {
-        "prompt_cache_min_tokens": 4096,
-        "deprecation_date": "2027-09-02",
         "cache_read_input_token_cost": 7.5e-08,
         "cache_read_input_token_cost_batches": 3.75e-08,
         "cache_read_input_token_cost_flex": 3.75e-08,
+        "cache_read_input_token_cost_priority": 1.35e-07,
+        "google_maps_grounding_cost_per_query": 0.014,
         "input_cost_per_token": 7.5e-07,
         "input_cost_per_token_batches": 3.75e-07,
         "input_cost_per_token_flex": 3.75e-07,
+        "input_cost_per_token_priority": 1.35e-06,
         "litellm_provider": "vertex_ai",
         "max_input_tokens": 1048576,
         "max_output_tokens": 65536,
@@ -29743,7 +29750,10 @@
         "output_cost_per_token": 3.75e-06,
         "output_cost_per_token_batches": 1.875e-06,
         "output_cost_per_token_flex": 1.875e-06,
+        "output_cost_per_token_priority": 6.75e-06,
+        "prompt_cache_min_tokens": 4096.0,
         "regional_endpoint_uplift_multiplier": 1.1,
+        "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
         "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -29761,11 +29771,12 @@
         ],
         "supports_audio_input": true,
         "supports_function_calling": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
-        "supports_minimal_reasoning_effort": false,
         "supports_response_schema": true,
         "supports_system_messages": true,
         "supports_tool_choice": true,
@@ -29773,17 +29784,7 @@
         "supports_video_input": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_native_streaming": true,
-        "input_cost_per_token_priority": 1.35e-06,
-        "output_cost_per_token_priority": 6.75e-06,
-        "cache_read_input_token_cost_priority": 1.35e-07,
-        "search_context_cost_per_query": {
-            "search_context_size_low": 0.014,
-            "search_context_size_medium": 0.014,
-            "search_context_size_high": 0.014
-        },
-        "web_search_billing_unit": "per_query",
-        "google_maps_grounding_cost_per_query": 0.014
+        "web_search_billing_unit": "per_query"
     },
     "vertex_ai/gemini-3.8-flash-cyber": {
         "cache_read_input_token_cost": 1.5e-07,
@@ -31989,14 +31990,15 @@
         "google_maps_grounding_cost_per_query": 0.014
     },
     "gemini-3.8-flash": {
-        "prompt_cache_min_tokens": 4096,
-        "deprecation_date": "2027-09-02",
         "cache_read_input_token_cost": 7.5e-08,
         "cache_read_input_token_cost_batches": 3.75e-08,
         "cache_read_input_token_cost_flex": 3.75e-08,
+        "cache_read_input_token_cost_priority": 1.35e-07,
+        "google_maps_grounding_cost_per_query": 0.014,
         "input_cost_per_token": 7.5e-07,
         "input_cost_per_token_batches": 3.75e-07,
         "input_cost_per_token_flex": 3.75e-07,
+        "input_cost_per_token_priority": 1.35e-06,
         "litellm_provider": "vertex_ai-language-models",
         "max_input_tokens": 1048576,
         "max_output_tokens": 65536,
@@ -32006,6 +32008,9 @@
         "output_cost_per_token": 3.75e-06,
         "output_cost_per_token_batches": 1.875e-06,
         "output_cost_per_token_flex": 1.875e-06,
+        "output_cost_per_token_priority": 6.75e-06,
+        "prompt_cache_min_tokens": 4096.0,
+        "search_context_cost_per_query": {"search_context_size_low":0.014,"search_context_size_medium":0.014,"search_context_size_high":0.014},
         "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -32021,14 +32026,15 @@
         "supported_output_modalities": [
             "text"
         ],
-        "supports_audio_output": false,
         "supports_audio_input": true,
+        "supports_audio_output": false,
         "supports_function_calling": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
-        "supports_minimal_reasoning_effort": false,
         "supports_response_schema": true,
         "supports_system_messages": true,
         "supports_tool_choice": true,
@@ -32036,17 +32042,7 @@
         "supports_video_input": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_native_streaming": true,
-        "input_cost_per_token_priority": 1.35e-06,
-        "output_cost_per_token_priority": 6.75e-06,
-        "cache_read_input_token_cost_priority": 1.35e-07,
-        "search_context_cost_per_query": {
-            "search_context_size_low": 0.014,
-            "search_context_size_medium": 0.014,
-            "search_context_size_high": 0.014
-        },
-        "web_search_billing_unit": "per_query",
-        "google_maps_grounding_cost_per_query": 0.014
+        "web_search_billing_unit": "per_query"
     },
     "gemini-3.8-flash-cyber": {
         "cache_read_input_token_cost": 1.5e-07,
@@ -32533,6 +32529,7 @@
     "github_copilot/claude-opus-4.8": {
         "cache_creation_input_token_cost": 6.25e-06,
         "cache_read_input_token_cost": 5e-07,
+        "deprecation_date": "2026-11-16",
         "input_cost_per_token": 5e-06,
         "litellm_provider": "github_copilot",
         "max_input_tokens": 200000,
@@ -33360,9 +33357,9 @@
         "supports_response_schema": true,
         "supports_vision": true,
         "supports_reasoning": true,
-        "supports_none_reasoning_effort": false,
-        "supports_xhigh_reasoning_effort": false,
-        "supports_minimal_reasoning_effort": true
+        "supports_none_reasoning_effort": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_minimal_reasoning_effort": false
     },
     "chatgpt/gpt-5.3-codex-spark": {
         "litellm_provider": "chatgpt",
@@ -33693,24 +33690,28 @@
         "output_cost_per_token": 1.2e-06
     },
     "baseten/MiniMaxAI/MiniMax-M2.5": {
+        "deprecation_date": "2026-10-16",
         "input_cost_per_token": 3e-07,
         "litellm_provider": "baseten",
         "mode": "chat",
         "output_cost_per_token": 1.2e-06
     },
     "baseten/nvidia/Nemotron-120B-A12B": {
+        "deprecation_date": "2026-10-16",
         "input_cost_per_token": 3e-07,
         "litellm_provider": "baseten",
         "mode": "chat",
         "output_cost_per_token": 7.5e-07
     },
     "baseten/zai-org/GLM-5": {
+        "deprecation_date": "2026-10-16",
         "input_cost_per_token": 9.5e-07,
         "litellm_provider": "baseten",
         "mode": "chat",
         "output_cost_per_token": 3.15e-06
     },
     "baseten/zai-org/GLM-4.7": {
+        "deprecation_date": "2026-10-16",
         "cache_read_input_token_cost": 1.2e-07,
         "input_cost_per_token": 6e-07,
         "litellm_provider": "baseten",
@@ -33728,24 +33729,28 @@
         "supports_vision": false
     },
     "baseten/zai-org/GLM-4.6": {
+        "deprecation_date": "2026-10-16",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "baseten",
         "mode": "chat",
         "output_cost_per_token": 2.2e-06
     },
     "baseten/moonshotai/Kimi-K2.5": {
+        "deprecation_date": "2026-10-16",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "baseten",
         "mode": "chat",
         "output_cost_per_token": 3e-06
     },
     "baseten/moonshotai/Kimi-K2-Thinking": {
+        "deprecation_date": "2026-10-16",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "baseten",
         "mode": "chat",
         "output_cost_per_token": 2.5e-06
     },
     "baseten/moonshotai/Kimi-K2-Instruct-0905": {
+        "deprecation_date": "2026-10-16",
         "input_cost_per_token": 6e-07,
         "litellm_provider": "baseten",
         "mode": "chat",
@@ -33769,12 +33774,14 @@
         "supports_vision": false
     },
     "baseten/deepseek-ai/DeepSeek-V3.1": {
+        "deprecation_date": "2026-10-16",
         "input_cost_per_token": 5e-07,
         "litellm_provider": "baseten",
         "mode": "chat",
         "output_cost_per_token": 1.5e-06
     },
     "baseten/deepseek-ai/DeepSeek-V3-0324": {
+        "deprecation_date": "2026-10-16",
         "input_cost_per_token": 7.7e-07,
         "litellm_provider": "baseten",
         "mode": "chat",
@@ -37805,9 +37812,9 @@
         "supports_tool_search": true,
         "supports_vision": true,
         "supports_web_search": true,
-        "supports_none_reasoning_effort": false,
-        "supports_xhigh_reasoning_effort": false,
-        "supports_minimal_reasoning_effort": true,
+        "supports_none_reasoning_effort": true,
+        "supports_xhigh_reasoning_effort": true,
+        "supports_minimal_reasoning_effort": false,
         "deprecation_date": "2027-04-01"
     },
     "gpt-5.3-chat-latest": {
@@ -45977,7 +45984,7 @@
         "deprecation_date": "2027-03-30",
         "input_cost_per_token": 4.5e-07,
         "litellm_provider": "bedrock_converse",
-        "max_input_tokens": 131072,
+        "max_input_tokens": 262144,
         "max_output_tokens": 16384,
         "max_tokens": 16384,
         "mode": "chat",
@@ -68129,7 +68136,7 @@
         "output_cost_per_token": 1.8e-06,
         "output_cost_per_token_flex": 9e-07,
         "litellm_provider": "bedrock_mantle",
-        "max_input_tokens": 128000,
+        "max_input_tokens": 262144,
         "max_output_tokens": 16000,
         "max_tokens": 16000,
         "mode": "chat",
@@ -68140,7 +68147,7 @@
         ],
         "supports_function_calling": true,
         "supports_tool_choice": true,
-        "source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrock/current/us-east-1/index.json"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-480b-a35b-instruct.html"
     },
     "bedrock_mantle/qwen.qwen3-next-80b-a3b-instruct": {
         "input_cost_per_token": 1.4e-07,
@@ -69622,7 +69629,7 @@
         "supports_max_reasoning_effort": false,
         "supports_minimal_reasoning_effort": false,
         "supports_native_streaming": true,
-        "supports_none_reasoning_effort": true,
+        "supports_none_reasoning_effort": false,
         "supports_parallel_function_calling": true,
         "supports_pdf_input": true,
         "supports_prompt_caching": true,
@@ -72134,6 +72141,7 @@
         "supports_vision": true
     },
     "baseten/moonshotai/Kimi-K2.6": {
+        "deprecation_date": "2026-10-16",
         "cache_read_input_token_cost": 1.6e-07,
         "input_cost_per_token": 9.5e-07,
         "litellm_provider": "baseten",
@@ -72151,6 +72159,7 @@
         "supports_vision": true
     },
     "baseten/moonshotai/Kimi-K2.7-Code": {
+        "deprecation_date": "2026-10-16",
         "cache_read_input_token_cost": 1.6e-07,
         "input_cost_per_token": 9.5e-07,
         "litellm_provider": "baseten",
@@ -72202,6 +72211,7 @@
         "supports_vision": false
     },
     "baseten/thinkingmachines/inkling": {
+        "deprecation_date": "2026-10-16",
         "cache_read_input_token_cost": 1.7e-07,
         "input_cost_per_token": 1e-06,
         "litellm_provider": "baseten",
@@ -72219,6 +72229,7 @@
         "supports_vision": true
     },
     "baseten/thinkingmachines/inkling-small": {
+        "deprecation_date": "2026-10-16",
         "cache_read_input_token_cost": 1e-07,
         "input_cost_per_token": 5e-07,
         "litellm_provider": "baseten",
@@ -72287,6 +72298,7 @@
         "supports_vision": false
     },
     "baseten/deepseek-ai/DeepSeek-V4-Pro": {
+        "deprecation_date": "2026-10-16",
         "cache_read_input_token_cost": 1.45e-07,
         "input_cost_per_token": 1.74e-06,
         "litellm_provider": "baseten",
@@ -75491,7 +75503,6 @@
         "web_search_billing_unit": "per_query"
     },
     "vertex_ai/gemini-4-argon": {
-        "deprecation_date": "2027-10-13",
         "cache_read_input_token_cost": 1e-07,
         "cache_read_input_token_cost_batches": 5e-08,
         "cache_read_input_token_cost_flex": 5e-08,
@@ -75960,11 +75971,11 @@
         "output_cost_per_token": 1.2e-06,
         "output_cost_per_token_priority": 1.5e-06,
         "source": "https://api.fireworks.ai/v1/serverless/models?format=nested",
-        "supports_function_calling": false,
+        "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
-        "supports_tool_choice": false,
+        "supports_tool_choice": true,
         "supports_vision": true
     },
     "github_copilot/claude-sonnet-4.6": {
@@ -76075,5 +76086,75 @@
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_vision": true
+    },
+    "together_ai/deepseek-ai/DeepSeek-V4.1-Flash-1017": {
+        "cache_read_input_token_cost": 6e-09,
+        "input_cost_per_token": 3e-07,
+        "litellm_provider": "together_ai",
+        "max_input_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 1.2e-06,
+        "source": "https://api.together.ai/v1/models",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "meta/muse-image-1.0": {
+        "litellm_provider": "meta",
+        "mode": "image_generation",
+        "output_cost_per_image": 0.01,
+        "source": "https://dev.meta.ai/docs/pricing-rate-limits",
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "image"
+        ]
+    },
+    "fireworks_ai/accounts/fireworks/models/minimax-m3p1-flash": {
+        "cache_read_input_token_cost": 1.5e-08,
+        "cache_read_input_token_cost_priority": 2.25e-08,
+        "input_cost_per_token": 1.5e-07,
+        "input_cost_per_token_priority": 2.25e-07,
+        "litellm_provider": "fireworks_ai",
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 1048576,
+        "max_tokens": 1048576,
+        "mode": "chat",
+        "output_cost_per_token": 6e-07,
+        "output_cost_per_token_priority": 9e-07,
+        "source": "https://platform.minimax.io/docs/guides/pricing",
+        "supports_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true
+    },
+    "meta/sam-3.1": {
+        "input_cost_per_image": 0.0025,
+        "litellm_provider": "meta",
+        "mode": "responses",
+        "source": "https://dev.meta.ai/docs/pricing-rate-limits.md",
+        "supported_endpoints": [
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ]
     }
 }

litellm/passthrough/timeout_utils.py

@@ -5,7 +5,10 @@ from typing import Final
 
 from pydantic import TypeAdapter
 
-from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout
+from litellm.litellm_core_utils.request_timeout_resolver import (
+    get_configured_request_timeout,
+    get_service_tier_timeout,
+)
 
 DEFAULT_PASS_THROUGH_REQUEST_TIMEOUT_SECONDS: Final = 600.0
 
@@ -50,21 +53,27 @@ def resolve_llm_passthrough_timeout(
     Anthropic /v1/messages).
 
     Non-streaming precedence: kwargs timeout/request_timeout -> litellm_params
-    timeout/request_timeout -> router_timeout -> litellm.request_timeout (litellm_settings.request_timeout,
+    timeout_by_service_tier[kwargs service_tier] -> litellm_params timeout/request_timeout -> router_timeout -> litellm.request_timeout (litellm_settings.request_timeout,
     when explicitly set) -> general_settings.pass_through_request_timeout -> 600s default.
 
     Streaming (``kwargs["stream"]`` truthy) resolves ``stream_timeout`` at every level before
     any generic timeout, matching ``Router._get_stream_timeout`` on the completion route:
-    kwargs stream_timeout -> litellm_params stream_timeout -> router_stream_timeout, then the
-    non-streaming chain above.
+    kwargs stream_timeout -> litellm_params stream_timeout_by_service_tier[kwargs service_tier] ->
+    litellm_params stream_timeout -> router_stream_timeout, then the non-streaming chain above.
 
     Only the first set value is validated as seconds, so a value in a lower-precedence
     field never fails the call.
     """
     request: Final = kwargs if kwargs is not None else _NO_PARAMS
     deployment: Final = litellm_params if litellm_params is not None else _NO_PARAMS
+    service_tier: Final = request.get("service_tier")
     stream_candidates: Final = (
-        (request.get("stream_timeout"), deployment.get("stream_timeout"), router_stream_timeout)
+        (
+            request.get("stream_timeout"),
+            get_service_tier_timeout(service_tier, deployment, "stream_timeout_by_service_tier"),
+            deployment.get("stream_timeout"),
+            router_stream_timeout,
+        )
         if request.get("stream")
         else ()
     )
@@ -72,6 +81,7 @@ def resolve_llm_passthrough_timeout(
         *stream_candidates,
         request.get("timeout"),
         request.get("request_timeout"),
+        get_service_tier_timeout(service_tier, deployment, "timeout_by_service_tier"),
         deployment.get("timeout"),
         deployment.get("request_timeout"),
         router_timeout,

litellm/proxy/_types.py

@@ -105,6 +105,7 @@ from litellm.types.utils import (
     ProviderField,
     StandardCallbackDynamicParams,
     StandardLoggingGuardrailInformation,
+    StandardLoggingKubernetesPodRouting,
     StandardLoggingMCPToolCall,
     StandardLoggingModelInformation,
     StandardLoggingPayloadErrorInformation,
@@ -667,6 +668,7 @@ class LiteLLMRoutes(enum.Enum):
             "/model/update",
             "/model/delete",
             "/model/info",
+            "/model/{model_id}/kubernetes_pods",
             "/jwt/key/mapping/new",
             "/jwt/key/mapping/update",
             "/jwt/key/mapping/delete",
@@ -2774,6 +2776,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
         default=None,
         description="Serve the OpenAI pass-through WebSocket route, which relays frames to OpenAI under the proxy's own provider credential without reading them. Off by default.",
     )
+    vertex_ai_pass_through_drop_anthropic_betas: tuple[str, ...] | None = Field(
+        default=None,
+        description="anthropic-beta header values the Vertex AI pass-through routes (/vertex_ai/* and /vertex-ai/*) remove before calling Vertex, for a beta a Vertex Claude model rejects, whether the client sent them as anthropic-beta or x-pass-anthropic-beta. Every other value the client sent is forwarded unchanged, and the header is left out when no value remains. Unset forwards the header as sent.",
+    )
     transcribe_media_buckets: list[str] | None = Field(
         default=None,
         description="S3 bucket names that keys other than proxy admins may read media from and write transcripts to through the Amazon Transcribe pass-through. Unset means only proxy admins can start transcription jobs.",
@@ -2787,6 +2793,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
         None,
         description="Fine-grained control over which object types to load from the database when store_model_in_db is True. Available types: 'models', 'mcp', 'guardrails', 'vector_stores', 'pass_through_endpoints', 'prompts', 'model_cost_map', 'tools', 'config_overrides'. If not set, all objects are loaded (default behavior).",
     )
+    self_serve_budget_policy: Literal["disabled", "lower_only", "ceiling"] | None = Field(
+        None,
+        description="Lets a non-admin change max_budget or budget_limits on a key they created and still own via /key/update. 'disabled' (default) requires an admin for any budget change. 'lower_only' lets the owner add or lower limits. 'ceiling' also lets the owner raise or remove limits when they could create that key with the new budget via /key/generate.",
+    )
     user_mcp_management_mode: UserMCPManagementMode | None = Field(
         None,
         description="Controls how non-admin users interact with MCP servers in the dashboard. 'restricted' shows only accessible servers, 'view_all' lists every server in read-only mode.",
@@ -3678,6 +3688,7 @@ class SpendLogsMetadata(TypedDict):
     mcp_tool_call_metadata: StandardLoggingMCPToolCall | None
     vector_store_request_metadata: list[StandardLoggingVectorStoreRequest] | None
     routing_decision: StandardLoggingRoutingDecision | None
+    kubernetes_pod_routing: ReadOnly[StandardLoggingKubernetesPodRouting | None]
     internal_call_origin: InternalCallOrigin | None
     litellm_roi_estimator: ReadOnly[NotRequired[bool | None]]
     guardrail_information: list[StandardLoggingGuardrailInformation] | None

litellm/proxy/guardrails/guardrail_endpoints.py

@@ -2367,9 +2367,6 @@ class _GuardrailLoggingObj(Protocol):
     @property
     def async_success_handler(self) -> "Callable[..., Awaitable[object]]": ...
 
-    @property
-    def success_handler(self) -> "Callable[..., object]": ...
-
 
 class _GuardrailProxyLogging(Protocol):
     @property
@@ -2399,10 +2396,6 @@ async def _emit_guardrail_success_logs(
     caller from receiving the guardrail response.  Returns the (possibly
     hook-modified) response.
     """
-    from litellm.litellm_core_utils.thread_pool_executor import (
-        executor as thread_pool_executor,
-    )
-
     try:
         modified: Final = await proxy_logging_obj.post_call_success_hook(
             data=data,
@@ -2430,16 +2423,6 @@ async def _emit_guardrail_success_logs(
             )
         except Exception:
             verbose_proxy_logger.exception("apply_guardrail: async_success_handler failed")
-        try:
-            thread_pool_executor.submit(
-                litellm_logging_obj.success_handler,
-                response_for_logging,
-                start_time,
-                end_time,
-                False,
-            )
-        except Exception:
-            verbose_proxy_logger.exception("apply_guardrail: success_handler submit failed")
 
     return response
 

litellm/proxy/hooks/proxy_track_cost_callback.py

@@ -2,8 +2,11 @@ import asyncio
 import traceback
 from collections.abc import Callable, Mapping, Sequence
 from datetime import datetime
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Protocol, cast
 
+from pydantic import TypeAdapter, ValidationError
+
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.batches.batch_utils import batch_cost_is_final
@@ -17,6 +20,7 @@ from litellm.litellm_core_utils.core_helpers import (
 )
 from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
+from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.auth_checks import (
     get_key_object,
@@ -100,6 +104,37 @@ def _proxy_stamped_used_client_oauth_token(
     return stamped if isinstance(stamped, bool) else None
 
 
+def _str_keyed_fields(value: object) -> Mapping[str, object]:
+    return value if is_str_keyed_mapping(value) else {}
+
+
+def _is_model_group(value: object) -> bool:
+    return isinstance(value, str) and bool(value)
+
+
+_TAG_LIST: Final = TypeAdapter(list[str])
+
+
+def _is_tag_list(value: object) -> bool:
+    try:
+        return bool(_TAG_LIST.validate_python(value, strict=True))
+    except ValidationError:
+        return False
+
+
+_ROUTED_METADATA_FIELD_CHECKS: Final = MappingProxyType({"model_group": _is_model_group, "tags": _is_tag_list})
+
+
+def _routed_metadata_fallback(request_data: Mapping[str, object]) -> Mapping[str, object]:
+    caller_metadata: Final = _str_keyed_fields(request_data.get("metadata"))
+    routed_metadata: Final = _str_keyed_fields(request_data.get("litellm_metadata"))
+    return {
+        field: routed_metadata[field]
+        for field, is_valid in _ROUTED_METADATA_FIELD_CHECKS.items()
+        if not caller_metadata.get(field) and is_valid(routed_metadata.get(field))
+    }
+
+
 def _failure_snapshot_with_guardrails_recorded_after_it(
     failure_snapshot: object, recorded_guardrail_information: object
 ) -> Mapping[str, object] | None:
@@ -220,10 +255,11 @@ class _ProxyDBLogger(CustomLogger):
             metadata=_metadata, original_exception=original_exception
         )
 
-        _metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_data, request_route)
+        request_fields: Final = _str_keyed_fields(request_data)
+        _metadata["used_client_oauth_token"] = _proxy_stamped_used_client_oauth_token(request_fields, request_route)
 
         existing_metadata: Final[dict] = request_data.get("metadata", None) or {}
-        existing_metadata.update(_metadata)
+        existing_metadata.update({**_routed_metadata_fallback(request_fields), **_metadata})
 
         litellm_metadata_bucket: Final = request_data.get("litellm_metadata")
         if (
@@ -471,11 +507,7 @@ class _ProxyDBLogger(CustomLogger):
                 await _release_budget_reservation(budget_reservation=budget_reservation)
                 # Non-model call types (health checks, afile_delete) have no model or standard_logging_object.
                 # Use .get() for "stream" to avoid KeyError on health checks.
-                # WS session wrappers (_aresponses_websocket, _arealtime) also reach here with
-                # result=None; their per-turn costs are tracked on the inner aresponses/realtime calls.
-                if sl_object is None and (
-                    not kwargs.get("model") or kwargs.get("call_type") in ("_aresponses_websocket", "_arealtime")
-                ):
+                if sl_object is None and (not kwargs.get("model") or kwargs.get("call_type") == "_arealtime"):
                     verbose_proxy_logger.warning(
                         "Cost tracking - skipping, no standard_logging_object for call_type=%s",
                         kwargs.get("call_type", "unknown"),

litellm/proxy/litellm_pre_call_utils.py

@@ -1672,11 +1672,9 @@ class LiteLLMProxyRequestSetup:
             metadata_from_headers["agent_id"] = agent_id_from_header
             verbose_proxy_logger.debug("Extracted agent_id from header: %s", agent_id_from_header)
 
-        spend_log_session_id: Final = (
+        caller_session_beside_trace: Final = (
             _caller_session_beside_trace_header(normalized_headers, request_body, _metadata_variable_name)
-            if litellm.request_correlation_in_logs
-            and chain_id is not None
-            and chain_id == normalized_headers.get("x-litellm-trace-id")
+            if chain_id is not None and chain_id == normalized_headers.get("x-litellm-trace-id")
             else None
         )
         if chain_id:
@@ -1738,8 +1736,8 @@ class LiteLLMProxyRequestSetup:
 
         if isinstance(data[_metadata_variable_name], dict):
             data[_metadata_variable_name].update(
-                {**metadata_from_headers, CALLER_SESSION_ID_METADATA_KEY: spend_log_session_id}
-                if spend_log_session_id
+                {**metadata_from_headers, CALLER_SESSION_ID_METADATA_KEY: caller_session_beside_trace}
+                if caller_session_beside_trace
                 else metadata_from_headers
             )
         return data