Merge remote-tracking branch 'origin/main' into litellm_router_stub_mypy_ci_guard

# Conflicts:
#	litellm/llms/bedrock/passthrough/guardrail_translation/handler.py

.github/workflows/test-unit.yml

@@ -1035,9 +1035,9 @@ jobs:
         run: |
           diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json
           if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
           else
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
             uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
             cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
             uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))"
@@ -1294,7 +1294,7 @@ jobs:
       - name: Install dependencies
         if: steps.changes.outputs.decision != 'skip'
         run: |
-          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router
+          .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard
 
       - name: Cache Prisma binaries
         if: steps.changes.outputs.decision != 'skip'
@@ -1497,9 +1497,9 @@ jobs:
           RUST_BRIDGE_ARTIFACT: ${{ needs.rust-bridge.outputs.artifact }}
         run: |
           if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils
           else
-            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
+            .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra switchyard --extra saml --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project --no-install-package litellm-proxy
             uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl
             cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so
           fi

AGENTS.md

@@ -52,7 +52,6 @@ If you ever write any human-facing text (pull requests, issues, commit messages,
 - don't use "—". Don't use ";" or ":" unless it would be nonsensical not to. Instead, reach for ",", ".", conjunction words, etc. in descending order of preference: vary among them, weighted toward the front of the list, and skip "," where it would cause a comma splice or the sentence is getting long. Overusing any one of them also feels AI-y. A word cap does not penalize you for adding more sentences
 - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc.
 - unless explicitly asked, don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose
-- don't add a trailing "." at the end of paragraphs (just like this file). That means every paragraph, not just the last one (of the markdown file, PR description, GitHub comment, etc.). Rule of thumb: if you're adding new line(s) before the next sentence, don't add a ".". Exception is that docs (i.e., litellm-docs) should have trailing "."
 - don't use →. Instead, prefer not to use arrows, and if need be, use -> instead
 - use plain, simple, everyday engineering language: the common phrase engineers actually say over rare compact phrasing, in grammatically complete sentences. When explicitly asked to use bullets or ordered lists and structure legitimately helps the reader, prefer nested bullets (any depth is fine) over dense lines in a flat structure
 

cookbook/ai_coding_tool_guides/chatgpt_subscription_passthrough.md

@@ -46,7 +46,43 @@ For a managed workspace rollout, use OpenAI's [gateway configuration guide](http
 
 ## Native compatibility and Pi
 
-For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials but does not need a model deployment or resolve model aliases. Restricted keys must allow the upstream model name.
+For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials. A ChatGPT model keeps its upstream name and needs no deployment of its own.
+
+### Mix ChatGPT and LiteLLM models
+
+On the native route, Codex's model picker shows OpenAI's catalog first and then the other chat and responses models the virtual key can use. Pick one of those and Codex's requests go through LiteLLM's `/v1/responses` with the key's usual routing, budgets, guardrails, and logging. The proxy removes the ChatGPT bearer and account header from that request, so the other provider and your callbacks never receive them. A name is routed this way when another provider serves it as an exact `model_name`, through a router `model_group_alias`, or as a team's public model name. A ChatGPT model, a name that only a wildcard deployment matches, or a name OpenAI's catalog lists for the caller's ChatGPT account keeps the passthrough, so a model OpenAI adds later still uses the subscription even when a deployment of another provider has the same name. The proxy learns the account's list from the catalog Codex fetches, or asks OpenAI with the caller's login. When it cannot get the list, for example because OpenAI is down, the request times out, or the client is not Codex, every request keeps the passthrough for the next minute, as it did before LiteLLM models were added to the picker. When the key can use many models, the picker lists the ones that fit in Codex's catalog size limit, and the proxy logs a warning with how many it left out.
+
+Codex's picker sends a `none` reasoning effort for a LiteLLM model whose catalog entry lists no effort levels. With `litellm_settings: drop_params: true`, the proxy drops it for a model that does not support `none`, and the provider applies its own default. A model whose cost map entry or `model_info` sets `supports_none_reasoning_effort: true` keeps it. Without `drop_params`, the effort reaches the provider unchanged.
+
+Codex also sends its ChatGPT app tools with every turn, which on a LiteLLM model adds a few hundred thousand prompt tokens to each turn. Set `apps = false` under `[features]` in Codex's `config.toml` to turn them off.
+
+Only a key that may call `/v1/responses` gets these models. A key whose `allowed_routes` covers only the native routes sees OpenAI's catalog unchanged, and every request it sends uses the subscription.
+
+Codex sends earlier reasoning items back with each turn. The proxy tags the reasoning that a LiteLLM model returns on this route, OpenAI and Azure deployments included. When a session switches to a ChatGPT model, the proxy drops those items before forwarding the request to OpenAI, which would reject them. Messages, tool calls, and ChatGPT's own reasoning are kept, and a LiteLLM model still gets its own reasoning back.
+
+When the catalog adds LiteLLM models for a key, its `ETag` is a digest of OpenAI's ETag and those models, and that key's responses carry the same value in `X-Models-Etag`. Codex keeps its cached catalog while the two match and fetches it again when OpenAI's catalog or the key's models change. A key whose catalog adds no LiteLLM models keeps OpenAI's ETags.
+
+### Restrict native models with an access group
+
+A key with a model allowlist needs a rule for ChatGPT names that OpenAI adds over time. Put the subscription wildcards in an access group under `chatgpt/` names:
+
+```yaml
+model_list:
+  - model_name: chatgpt/gpt-*
+    litellm_params:
+      model: chatgpt/gpt-*
+    model_info:
+      access_groups: ["chatgpt-native"]
+  - model_name: chatgpt/codex-*
+    litellm_params:
+      model: chatgpt/codex-*
+    model_info:
+      access_groups: ["chatgpt-native"]
+```
+
+Then add `chatgpt-native` to the key's models, next to any other model names it should reach. On the native route, a key that holds only the `chatgpt/` form of a name is checked as that form, so a request for `gpt-5.6-sol` is allowed when the key may call `chatgpt/gpt-5.6-sol`. The key can call every ChatGPT model that matches the wildcards, including ones the proxy configuration does not name.
+
+On `/v1` these wildcards match only a request that asks for a `chatgpt/` name, so a request for a plain `gpt-` or `codex-` name still goes where it went before, such as a `*` catch-all deployment. A `/v1` request for a `chatgpt/` name without a ChatGPT login falls back to the proxy's own saved login if it has one, so keep that login unset as described above.
 
 ### Configure Pi
 
@@ -71,7 +107,7 @@ Keep Pi's subscription login as the provider credential. The header value names
 
 Send a prompt and check the completed request in LiteLLM's Logs page. Its gateway user/key and token counts should match the caller. Give each person their own LiteLLM virtual key for separate attribution. The gateway credential and ChatGPT login are independently authenticated; LiteLLM does not establish that both identities belong to the same person.
 
-The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key must allow that name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions.
+The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key needs the `chatgpt-native` access group above or that exact name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions.
 
 The proxy forwards the ChatGPT bearer and account header to the fixed OpenAI Codex endpoint. It removes gateway credentials and cookies from the upstream request and redacts credential headers in logging callbacks. The client remains responsible for refresh and reauthentication. Native passthrough returns authentication and quota errors without switching subscriptions. Standard model routes retain the caller login across ChatGPT retries; explicitly configured cross-provider fallbacks use their own credentials. An explicit workspace header is preserved for OpenAI to validate.
 

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.81"
+version = "0.1.83"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.81"
+version = "0.1.83"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.112"
+version = "0.4.114"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -30,7 +30,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.112"
+version = "0.4.114"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/crates/python-bridge/src/router/host.rs

@@ -20,10 +20,10 @@ use std::{
 
 use async_trait::async_trait;
 use litellm_router_core::{
-    Attempt, AttemptError, Blocked, BuildAsk, BuildPath, Built, Deployment, Error, Host, Output,
-    Payload, PreRoute, PreRouted, Provided, Randomness, RouterCache, RouterEvent, RouterRequest,
-    RouterResponse, RoutingContext, SecretRef, SelectStage, ThreadRandomness,
-    build::BuiltDeployment, host::ModelInfoLookup, model_info::MergedModelInfo,
+    Attempt, AttemptError, Blocked, BuildAsk, BuildPath, Built, CallbackFiltered, Deployment,
+    Error, Host, Output, Payload, PreRoute, PreRouted, Provided, Randomness, RouterCache,
+    RouterEvent, RouterRequest, RouterResponse, RoutingContext, SecretRef, SelectStage,
+    ThreadRandomness, build::BuiltDeployment, host::ModelInfoLookup, model_info::MergedModelInfo,
     precall::PreCallFacts,
 };
 use pyo3::{PyResult, Python, types::PyAnyMethods};
@@ -298,8 +298,9 @@ pub(crate) enum HostCallResponse {
     PreCall(PreCallFacts),
     /// The candidate models a `routing_plugins` select step reported.
     Models(Option<Vec<String>>),
-    /// The ids a `callback_filters` select step left, or the error a filter raised.
-    Filtered(Result<Vec<String>, AttemptError>),
+    /// The ids a `callback_filters` select step left with the one a filter prefers, or the error
+    /// a filter raised.
+    Filtered(Result<(Vec<String>, Option<String>), AttemptError>),
     /// What a pick's select or pre-route step, or a fallback check, raised on the host instead of
     /// answering: the walk takes it as the try's failure.
     Raised(AttemptError),
@@ -352,8 +353,10 @@ impl HostCallResponse {
                 got: self.kind(),
             });
         }
-        if let (Self::Select(ids) | Self::Filtered(Ok(ids)), HostCall::Select { candidates, .. }) =
-            (self, call)
+        if let (
+            Self::Select(ids) | Self::Filtered(Ok((ids, _))),
+            HostCall::Select { candidates, .. },
+        ) = (self, call)
             && let Some(id) = ids.iter().find(|id| !candidates.contains(id))
         {
             return Err(DriveError::UnknownCandidate { id: id.clone() });
@@ -890,7 +893,7 @@ impl Host for StepHost {
         &self,
         ctx: &RoutingContext<'_>,
         candidates: Vec<Arc<Deployment>>,
-    ) -> Result<Vec<Arc<Deployment>>, Error> {
+    ) -> Result<CallbackFiltered, Error> {
         let (scope, group) = ctx.scope.wire();
         let joined = |own: &[String], walked: &[String]| [own, walked].concat();
         let call = HostCall::Select {
@@ -909,8 +912,11 @@ impl Host for StepHost {
         let HostCallResponse::Filtered(result) = call_host(call).await else {
             unreachable!("{CHECKED}");
         };
-        let ids = result.map_err(Error::Attempt)?;
-        Ok(answered_by_id(&candidates, &ids))
+        let (ids, preferred) = result.map_err(Error::Attempt)?;
+        Ok(CallbackFiltered {
+            kept: answered_by_id(&candidates, &ids),
+            preferred,
+        })
     }
 
     async fn allow_fallback(

litellm-rust/crates/python-bridge/src/router/mod.rs

@@ -1181,7 +1181,8 @@ fn host_call_response(
             stage: Stage::CallbackFilters { .. },
             ..
         } => {
-            // The remaining ids, or a failure mapping as an attempt reports one.
+            // The remaining ids, alone or paired with the id a filter prefers, or a failure
+            // mapping as an attempt reports one.
             let outcome = outcome.ok_or_else(missing)?;
             if let Ok(failure) = outcome.cast::<PyDict>() {
                 return match attempt_outcome(failure)? {
@@ -1191,7 +1192,10 @@ fn host_call_response(
                     )),
                 };
             }
-            Ok(HostCallResponse::Filtered(Ok(outcome.extract()?)))
+            if let Ok(answer) = outcome.cast::<PyTuple>() {
+                return Ok(HostCallResponse::Filtered(Ok(answer.extract()?)));
+            }
+            Ok(HostCallResponse::Filtered(Ok((outcome.extract()?, None))))
         }
         HostCall::Select {
             stage: Stage::Strategy { records, .. },

litellm-rust/crates/router-core/src/error.rs

@@ -284,6 +284,7 @@ impl Error {
     strum::EnumString,
     strum::IntoStaticStr,
     strum::VariantArray,
+    serde_with::SerializeDisplay,
 )]
 pub enum FailureKind {
     #[strum(serialize = "RateLimitError")]
@@ -471,6 +472,17 @@ mod tests {
         assert_eq!(<&'static str>::from(kind), wire);
     }
 
+    #[rstest]
+    #[case::rate_limit(FailureKind::RateLimit, "RateLimitError")]
+    #[case::timeout(FailureKind::Timeout, "Timeout")]
+    #[case::api_connection(FailureKind::ApiConnection, "APIConnectionError")]
+    fn failure_kind_serializes_as_its_exception_name(
+        #[case] kind: FailureKind,
+        #[case] wire: &str,
+    ) {
+        assert_eq!(serde_json::to_value(kind).unwrap(), serde_json::json!(wire));
+    }
+
     #[rstest]
     #[case::empty("")]
     #[case::lowercase("ratelimiterror")]

litellm-rust/crates/router-core/src/host.rs

@@ -342,7 +342,10 @@ pub struct Output {
 }
 
 /// How a provider blocked a response the call otherwise served.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::IntoStaticStr, strum::EnumString)]
+#[derive(
+    Clone, Copy, Debug, PartialEq, Eq, serde::Serialize, strum::IntoStaticStr, strum::EnumString,
+)]
+#[serde(rename_all = "snake_case")]
 #[strum(serialize_all = "snake_case")]
 pub enum Blocked {
     /// A chat response whose first choice was content-filtered (or that has no choices).
@@ -402,8 +405,16 @@ pub enum SelectStage {
     OverrideCheck(RoutingStrategy),
 }
 
-/// Routing events, delivered through [`Host::observe`].
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Debug)]
+pub struct CallbackFiltered {
+    pub kept: Vec<Arc<Deployment>>,
+    pub preferred: Option<String>,
+}
+
+/// Routing events, delivered through [`Host::observe`]. They serialize as JSON objects tagged by
+/// `event` (the variant name), durations in seconds.
+#[derive(Clone, Debug, PartialEq, serde::Serialize)]
+#[serde(tag = "event")]
 pub enum RouterEvent {
     /// A pick for `model` (the level's group) begins: its route phase opens.
     PickStarted {
@@ -413,6 +424,7 @@ pub enum RouterEvent {
         model_group: String,
         deployment_id: String,
         /// How long the pick took (`_acompletion`'s `async_get_available_deployment` service event).
+        #[serde(serialize_with = "seconds")]
         pick: Duration,
     },
     Succeeded {
@@ -425,11 +437,13 @@ pub enum RouterEvent {
     },
     CooldownSet {
         deployment_id: String,
+        #[serde(serialize_with = "seconds")]
         cooldown: Duration,
     },
     Retry {
         model_group: String,
         retry: u32,
+        #[serde(serialize_with = "seconds")]
         delay: Duration,
     },
     Fallback {
@@ -584,6 +598,13 @@ impl PickError {
     }
 }
 
+/// A pick's error serializes as its message.
+impl serde::Serialize for PickError {
+    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
+        serializer.collect_str(&self.0)
+    }
+}
+
 impl PartialEq for PickError {
     fn eq(&self, other: &Self) -> bool {
         format!("{:?}", self.0) == format!("{:?}", other.0)
@@ -591,7 +612,8 @@ impl PartialEq for PickError {
 }
 
 /// Which error a [`RouterEvent::RetryLogged`] records.
-#[derive(Clone, Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq, serde::Serialize)]
+#[serde(tag = "cause", content = "error", rename_all = "snake_case")]
 pub enum RetryCause {
     /// The last attempt's failure.
     Attempt,
@@ -698,14 +720,19 @@ pub trait Host: Send + Sync {
     }
 
     /// `async_callback_filter_deployments`: the host's routing callbacks narrow `candidates` in
-    /// turn. A filter that raises fails the try with [`Error::Attempt`], which the call retries and
-    /// falls back from like any attempt failure. A host without filters keeps every candidate.
+    /// turn, and may name one deployment they prefer, which the pick takes only when the filters
+    /// after the callbacks keep it. A filter that raises fails the try with [`Error::Attempt`],
+    /// which the call retries and falls back from like any attempt failure. A host without
+    /// filters keeps every candidate and prefers none.
     async fn callback_filters(
         &self,
         _ctx: &RoutingContext<'_>,
         candidates: Vec<Arc<Deployment>>,
-    ) -> Result<Vec<Arc<Deployment>>, Error> {
-        Ok(candidates)
+    ) -> Result<CallbackFiltered, Error> {
+        Ok(CallbackFiltered {
+            kept: candidates,
+            preferred: None,
+        })
     }
     /// The values of `refs`, one per ref, in order: what the deployment build substitutes for
     /// each `os.environ/` reference it found in entry `index` ([`crate::build`]). Asked once per
@@ -745,3 +772,83 @@ pub trait Host: Send + Sync {
         Ok(Value::Null)
     }
 }
+
+fn seconds<S: serde::Serializer>(duration: &Duration, serializer: S) -> Result<S::Ok, S::Error> {
+    serializer.serialize_f64(duration.as_secs_f64())
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::{Value, json};
+
+    use super::*;
+
+    #[rstest]
+    #[case::selected(
+        RouterEvent::Selected {
+            model_group: "gpt-4o".to_owned(),
+            deployment_id: "azure-eastus".to_owned(),
+            pick: Duration::from_millis(1500),
+        },
+        json!({"event": "Selected", "model_group": "gpt-4o", "deployment_id": "azure-eastus", "pick": 1.5})
+    )]
+    #[case::attempt_failed(
+        RouterEvent::AttemptFailed {
+            deployment_id: "openai".to_owned(),
+            kind: FailureKind::RateLimit,
+            status: Some(429),
+        },
+        json!({"event": "AttemptFailed", "deployment_id": "openai", "kind": "RateLimitError", "status": 429})
+    )]
+    #[case::cooldown_set(
+        RouterEvent::CooldownSet { deployment_id: "openai".to_owned(), cooldown: Duration::from_secs(30) },
+        json!({"event": "CooldownSet", "deployment_id": "openai", "cooldown": 30.0})
+    )]
+    #[case::retry(
+        RouterEvent::Retry { model_group: "gpt-4o".to_owned(), retry: 1, delay: Duration::ZERO },
+        json!({"event": "Retry", "model_group": "gpt-4o", "retry": 1, "delay": 0.0})
+    )]
+    #[case::blocked(
+        RouterEvent::Blocked { level: 0, model: "claude".to_owned(), blocked: Blocked::ContentFilter },
+        json!({"event": "Blocked", "level": 0, "model": "claude", "blocked": "content_filter"})
+    )]
+    #[case::retry_logged_after_an_attempt(
+        RouterEvent::RetryLogged {
+            scope: 0,
+            level: 0,
+            model: "gpt-4o".to_owned(),
+            attempted_retries: None,
+            cause: RetryCause::Attempt,
+        },
+        json!({
+            "event": "RetryLogged", "scope": 0, "level": 0, "model": "gpt-4o",
+            "attempted_retries": null, "cause": {"cause": "attempt"},
+        })
+    )]
+    #[case::retry_logged_after_a_pick(
+        RouterEvent::RetryLogged {
+            scope: 1,
+            level: 1,
+            model: "backup".to_owned(),
+            attempted_retries: Some(2),
+            cause: RetryCause::Pick(PickError(Arc::new(Error::UnknownModel { model: "backup".to_owned() }))),
+        },
+        json!({
+            "event": "RetryLogged", "scope": 1, "level": 1, "model": "backup", "attempted_retries": 2,
+            "cause": {"cause": "pick", "error": "no deployment serves model `backup`"},
+        })
+    )]
+    #[case::mock_failed(
+        RouterEvent::MockFailed {
+            mock: MockFailure::RateLimit,
+            level: 0,
+            model: "gpt-4o".to_owned(),
+            num_retries: None,
+        },
+        json!({"event": "MockFailed", "mock": "rate_limit", "level": 0, "model": "gpt-4o", "num_retries": null})
+    )]
+    fn an_event_serializes_tagged_by_its_name(#[case] event: RouterEvent, #[case] expected: Value) {
+        assert_eq!(serde_json::to_value(&event).unwrap(), expected);
+    }
+}

litellm-rust/crates/router-core/src/lib.rs

@@ -124,8 +124,8 @@ pub use crate::{
     error::{AttemptError, ConfigError, Error, FailureKind, Payload, Unsupported},
     health::{Clock, Health, Outcome, SystemClock},
     host::{
-        Attempt, Blocked, BuildAsk, BuildPath, Built, Host, Output, Provided, RetryCause,
-        RouterEvent, RouterRequest, RoutingContext, SecretAt, SecretRef, SelectStage,
+        Attempt, Blocked, BuildAsk, BuildPath, Built, CallbackFiltered, Host, Output, Provided,
+        RetryCause, RouterEvent, RouterRequest, RoutingContext, SecretAt, SecretRef, SelectStage,
     },
     outcome::{OutcomeCallbacks, OutcomeWrite},
     prerouting::{PreRoute, PreRouted},

litellm-rust/crates/router-core/src/mock.rs

@@ -7,7 +7,15 @@ use crate::error::{AttemptError, FailureKind};
 
 /// Which mock failure a request asks for.
 #[derive(
-    Clone, Copy, Debug, PartialEq, Eq, Deserialize, strum::IntoStaticStr, strum::EnumString,
+    Clone,
+    Copy,
+    Debug,
+    PartialEq,
+    Eq,
+    Deserialize,
+    serde::Serialize,
+    strum::IntoStaticStr,
+    strum::EnumString,
 )]
 #[serde(rename_all = "snake_case")]
 #[strum(serialize_all = "snake_case")]

litellm-rust/crates/router-core/src/select.rs

@@ -456,7 +456,7 @@ impl Core {
         let request_order = request.target_order.filter(|_| !inputs.fallback_hop);
         // `async_callback_filter_deployments` runs after the cooldown and blocked filters on every
         // async candidate list but a deployment named by id and a listed pass-through pick.
-        if settings.callback_filters
+        let preferred = if settings.callback_filters
             && !inputs.count_only
             && !resolution.kind.is_single()
             && request.pass_through != Some(PassThrough::Listed)
@@ -472,8 +472,12 @@ impl Core {
             ctx.excluded_ids = &inputs.excluded_ids;
             ctx.skipped_ids = &inputs.skipped_ids;
             ctx.target_order = inputs.target_order.or(request_order);
-            healthy = self.host.callback_filters(&ctx, healthy).await?;
-        }
+            let filtered = self.host.callback_filters(&ctx, healthy).await?;
+            healthy = filtered.kept;
+            filtered.preferred
+        } else {
+            None
+        };
         if inputs.count_only {
             return Ok(Candidates {
                 name: name.to_owned(),
@@ -648,8 +652,13 @@ impl Core {
             healthy.retain(|d| !d.cannot_run_batches());
         }
 
-        if let Some(pinned) = request.pinned_deployment_id.as_deref()
-            && healthy.iter().any(|d| d.id == pinned)
+        if let Some(pinned) = [
+            request.pinned_deployment_id.as_deref(),
+            preferred.as_deref(),
+        ]
+        .into_iter()
+        .flatten()
+        .find(|id| healthy.iter().any(|d| d.id == *id))
         {
             healthy.retain(|d| d.id == pinned);
         }

litellm-rust/crates/router-core/tests/call_loop.rs

@@ -17,7 +17,7 @@ use litellm_router_core::{
 };
 use parking_lot::Mutex;
 use rstest::{fixture, rstest};
-use serde_json::json;
+use serde_json::{Value, json};
 
 #[derive(Default)]
 struct ScriptedHost {
@@ -1809,3 +1809,55 @@ fn request_for(model: &str) -> RouterRequest {
         ..RouterRequest::default()
     }
 }
+
+#[rstest]
+#[tokio::test]
+async fn every_event_of_a_retried_and_fallen_back_call_serializes_tagged_by_its_name(
+    config: RouterConfig,
+) {
+    let host = Arc::new(ScriptedHost::failing(
+        "primary",
+        vec![failure(FailureKind::RateLimit, 429); 3],
+    ));
+    let router = router(
+        config,
+        json!({"num_retries": 1, "fallbacks": [{"primary": ["backup"]}]}),
+        host.clone(),
+    );
+    router
+        .call(Op::from("acompletion"), request())
+        .await
+        .unwrap();
+    let tags: Value = host
+        .events
+        .lock()
+        .iter()
+        .map(|event| serde_json::to_value(event).unwrap()["event"].clone())
+        .collect();
+    assert_eq!(
+        tags,
+        json!([
+            "RetriesOpened",
+            "PickStarted",
+            "Selected",
+            "AttemptFailed",
+            "CooldownSet",
+            "RetryBudget",
+            "RetryLogged",
+            "Retry",
+            "PickStarted",
+            "Selected",
+            "AttemptFailed",
+            "CooldownSet",
+            "RetryLogged",
+            "FallingBack",
+            "RetryLogged",
+            "Fallback",
+            "RetriesOpened",
+            "PickStarted",
+            "Selected",
+            "Succeeded",
+            "FallbackDone",
+        ])
+    );
+}

litellm/a2a_protocol/card_resolver.py

@@ -4,13 +4,14 @@ Custom A2A Card Resolver for LiteLLM.
 Extends the A2A SDK's card resolver to support multiple well-known paths.
 """
 
-from collections.abc import Mapping
+from collections.abc import Mapping, Sequence
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Protocol, runtime_checkable
 
 from litellm._logging import verbose_logger
 from litellm.a2a_protocol.exceptions import A2AAgentCardDiscoveryError
 from litellm.constants import LOCALHOST_URL_PATTERNS
+from litellm.litellm_core_utils.core_helpers import has_attributes
 
 if TYPE_CHECKING:
     from a2a.types import AgentCard
@@ -130,15 +131,34 @@ def normalize_agent_card_interfaces(agent_card: "AgentCard", protocol_version: s
     return normalized
 
 
+class _UrlCarrier(Protocol):
+    url: str
+
+
+class _InterfacesCarrier(Protocol):
+    @property
+    def supported_interfaces(self) -> Sequence[_UrlCarrier]: ...
+
+
+def _url_of(value: object) -> str | None:
+    return value.url if has_attributes(value, _UrlCarrier, "url") else None
+
+
+def _supported_interfaces(agent_card: object) -> Sequence[_UrlCarrier] | None:
+    if has_attributes(agent_card, _InterfacesCarrier, "supported_interfaces"):
+        return agent_card.supported_interfaces
+    return None
+
+
 def get_agent_card_url(agent_card: "AgentCard") -> str | None:
     """Return the agent endpoint URL from the resolved SDK card."""
-    url: Final = getattr(agent_card, "url", None)
+    url: Final = _url_of(agent_card)
     if url:
         return url
 
-    interfaces: Final = getattr(agent_card, "supported_interfaces", None)
+    interfaces: Final = _supported_interfaces(agent_card)
     if interfaces:
-        return getattr(interfaces[0], "url", None)
+        return _url_of(interfaces[0])
     return None
 
 
@@ -168,16 +188,16 @@ def fix_agent_card_url(agent_card: "AgentCard", base_url: str) -> "AgentCard":
     Returns:
         The agent card with the URL fixed if necessary
     """
-    card_url: Final = getattr(agent_card, "url", None)
+    card_url: Final = _url_of(agent_card)
 
     if card_url and is_localhost_or_internal_url(card_url):
         # Normalize base_url to ensure it ends with /
         fixed_url: Final = base_url.rstrip("/") + "/"
         agent_card.url = fixed_url
 
-    interfaces: Final = getattr(agent_card, "supported_interfaces", None)
+    interfaces: Final = _supported_interfaces(agent_card)
     if interfaces:
-        interface_url: Final = getattr(interfaces[0], "url", None)
+        interface_url: Final = _url_of(interfaces[0])
         if interface_url and is_localhost_or_internal_url(interface_url):
             interfaces[0].url = base_url.rstrip("/") + "/"
 
@@ -194,6 +214,9 @@ class LiteLLMA2ACardResolver(_A2ACardResolver):
     - /agentCard/v1.0
     """
 
+    if TYPE_CHECKING:
+        base_url: str
+
     async def get_agent_card(
         self,
         relative_card_path: str | None = None,

litellm/a2a_protocol/exception_mapping_utils.py

@@ -4,7 +4,7 @@ A2A Protocol Exception Mapping Utils.
 Maps A2A SDK exceptions to LiteLLM A2A exception types.
 """
 
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Final
 
 from litellm._logging import verbose_logger
 from litellm.a2a_protocol.card_resolver import (
@@ -21,6 +21,7 @@ from litellm.constants import CONNECTION_ERROR_PATTERNS
 
 if TYPE_CHECKING:
     from a2a.client import Client as A2AClientType
+    from a2a.types import AgentCard
 
 
 _A2A_KIND_DIALECT_HINT: Final = (
@@ -165,7 +166,7 @@ def map_a2a_exception(
 
 async def handle_a2a_localhost_retry(
     error: A2ALocalhostURLError,
-    agent_card: Any,
+    agent_card: "AgentCard | None",
     a2a_client: "A2AClientType",
     is_streaming: bool = False,
 ) -> "A2AClientType":

litellm/a2a_protocol/litellm_completion_bridge/handler.py

@@ -10,7 +10,7 @@ A2A Streaming Events (in order):
 4. Status update (kind: "status-update") - Final status "completed" with final=true
 """
 
-from collections.abc import AsyncIterator, Callable, Coroutine, Mapping
+from collections.abc import AsyncIterator, Mapping
 from types import MappingProxyType
 from typing import Any, Final
 
@@ -126,10 +126,7 @@ class A2ACompletionBridgeHandler:
 
     @staticmethod
     async def _acompletion(completion_params: Mapping[str, object]) -> ModelResponse | CustomStreamWrapper:
-        acompletion_fn: Final[Callable[..., Coroutine[object, object, ModelResponse | CustomStreamWrapper]]] = vars(
-            litellm
-        )["acompletion"]
-        return await acompletion_fn(**completion_params)
+        return await litellm.acompletion(**completion_params)
 
     @staticmethod
     async def handle_non_streaming(

litellm/a2a_protocol/main.py

@@ -14,7 +14,7 @@ import datetime
 import uuid
 from collections.abc import AsyncIterator, Coroutine, Mapping
 from types import MappingProxyType, ModuleType
-from typing import TYPE_CHECKING, Final, Optional, cast
+from typing import TYPE_CHECKING, Final, Optional, Protocol, cast
 
 from typing_extensions import TypedDict, Unpack
 
@@ -24,6 +24,7 @@ from litellm.a2a_protocol.streaming_iterator import A2AStreamingIterator
 from litellm.a2a_protocol.utils import A2ARequestUtils
 from litellm.constants import DEFAULT_A2A_AGENT_TIMEOUT
 from litellm.litellm_core_utils.asyncify import asyncify
+from litellm.litellm_core_utils.core_helpers import require_attributes
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.llms.custom_httpx.http_handler import (
     get_async_httpx_client,
@@ -36,7 +37,6 @@ if TYPE_CHECKING:
     from a2a.client import Client as A2AClientType
     from a2a.client import ClientCallContext as A2ACallContextType
     from a2a.compat.v0_3.types import (
-        AgentCard,
         Message,
         SendMessageRequest,
         SendMessageResponse,
@@ -45,6 +45,7 @@ if TYPE_CHECKING:
         SendStreamingMessageSuccessResponse,
         Task,
     )
+    from a2a.types import AgentCard
     from a2a.types.a2a_pb2 import SendMessageRequest as CoreSendMessageRequest
     from a2a.types.a2a_pb2 import StreamResponse as CoreStreamResponse
 
@@ -95,6 +96,12 @@ class _SendMessageOptions(TypedDict, total=False, extra_items=object):
     pass
 
 
+class _StreamingRequestConverter(Protocol):
+    def to_core_send_message_request(
+        self, compat_req: "SendStreamingMessageRequest", /
+    ) -> "CoreSendMessageRequest": ...
+
+
 def _set_usage_on_logging_obj(
     kwargs: Mapping[str, object],
     prompt_tokens: int,
@@ -375,7 +382,9 @@ async def _stream_messages(
             "The 'a2a' package is required for A2A agent invocation. Install it with: pip install a2a-sdk"
         )
 
-    pb_request: Final[CoreSendMessageRequest] = _a2a_conversions.to_core_send_message_request(request)
+    pb_request: Final[CoreSendMessageRequest] = require_attributes(
+        _a2a_conversions, _StreamingRequestConverter, "to_core_send_message_request"
+    ).to_core_send_message_request(request)
     async for event in a2a_client.send_message(pb_request, context=_get_a2a_call_context(a2a_client)):
         compat_chunk = _to_compat_stream_response(event, request_id=request.id)
         yield SendStreamingMessageResponse(root=compat_chunk)

litellm/caching/caching.py

@@ -13,10 +13,10 @@ import json
 import logging
 import time
 import traceback
-from collections.abc import Generator, Mapping
+from collections.abc import Awaitable, Callable, Generator, Mapping
 from contextlib import contextmanager
 from enum import Enum
-from typing import Any, Final, Literal, cast
+from typing import Any, Final, Literal, Protocol, cast
 
 from pydantic import BaseModel, ConfigDict, TypeAdapter
 
@@ -25,6 +25,7 @@ from litellm._internal_context import current_service_target, service_target
 from litellm._logging import verbose_logger
 from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
 from litellm.integrations.otel.runtime import phase_span
+from litellm.litellm_core_utils.core_helpers import require_attributes
 from litellm.litellm_core_utils.model_param_helper import (
     ModelParamHelper,
     embeds_thought_signature_in_tool_call_id,
@@ -52,6 +53,16 @@ _REDIS_STARTUP_NODES_ADAPTER: Final = TypeAdapter(
 )
 
 
+class _SupportsAsyncPing(Protocol):
+    @property
+    def ping(self) -> Callable[[], Awaitable[object]] | None: ...
+
+
+class _SupportsAsyncDeleteCacheKeys(Protocol):
+    @property
+    def delete_cache_keys(self) -> Callable[[object], Awaitable[object]] | None: ...
+
+
 def _keyed_provider_params_in_extra_body(kwargs: Mapping[str, object]) -> str:
     raw_extra_body: Final = kwargs.get("extra_body")
     if not isinstance(raw_extra_body, dict):
@@ -1032,14 +1043,16 @@ class Cache:
         cache_key, cached_data, kwargs = self._add_cache_logic(result=result, **kwargs)
         await self.cache.batch_cache_write(cache_key, cached_data, **kwargs)
 
-    async def ping(self):
-        cache_ping: Final = getattr(self.cache, "ping")
+    async def ping(self) -> object:
+        cache_ping: Final = require_attributes(self.cache, _SupportsAsyncPing, "ping").ping
         if cache_ping:
             return await cache_ping()
         return None
 
-    async def delete_cache_keys(self, keys):
-        cache_delete_cache_keys: Final = getattr(self.cache, "delete_cache_keys")
+    async def delete_cache_keys(self, keys: object) -> object:
+        cache_delete_cache_keys: Final = require_attributes(
+            self.cache, _SupportsAsyncDeleteCacheKeys, "delete_cache_keys"
+        ).delete_cache_keys
         if cache_delete_cache_keys:
             return await cache_delete_cache_keys(keys)
         return None

litellm/caching/caching_handler.py

@@ -22,6 +22,7 @@ from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable,
 from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar
 
 from pydantic import ConfigDict, SkipValidation, TypeAdapter, ValidationError
+from typing_extensions import TypedDict
 
 import litellm
 from litellm._internal_context import is_internal_call, post_response_phase
@@ -90,6 +91,10 @@ class CachingHandlerResponse(LiteLLMBaseModel):
 in_memory_cache_obj: Final = InMemoryCache()
 
 
+class _CacheCallKwargs(TypedDict, total=False, extra_items=object):
+    pass
+
+
 def _drop_logging_obj_from_kwargs(request_kwargs: dict[str, object]) -> dict[str, object]:
     """
     The caching handler is stored on the Logging object
@@ -455,7 +460,7 @@ class LLMCachingHandler:
         if litellm.cache is not None and self._is_call_type_supported_by_cache(original_function=original_function):
             args = args or ()
             # Now that we confirmed caching will happen, prepare kwargs
-            new_kwargs: Final = kwargs.copy()
+            new_kwargs: Final = dict[str, object](kwargs)
             new_kwargs.update(
                 convert_args_to_kwargs(
                     self.original_function,
@@ -470,7 +475,7 @@ class LLMCachingHandler:
             print_verbose("Checking Sync Cache")
             logging_obj.reset_caching_details()
             with response_cache_phase("get"):
-                cached_result = litellm.cache.get_cache(**new_kwargs)
+                cached_result = litellm.cache.get_cache(**_CacheCallKwargs(**new_kwargs))
             if is_response_without_output(cached_result):
                 verbose_logger.debug("LiteLLM Cache: cached response has no output, treating it as a miss")
                 return CachingHandlerResponse(cached_result=None)
@@ -480,7 +485,7 @@ class LLMCachingHandler:
                     pass
                 else:
                     call_type = original_function.__name__
-                    cached_result = self._convert_cached_result_to_model_response(
+                    converted_result: Final = self._convert_cached_result_to_model_response(
                         cached_result=cached_result,
                         call_type=call_type,
                         kwargs=kwargs,
@@ -508,16 +513,16 @@ class LLMCachingHandler:
                         logging_obj=logging_obj,
                         model=f"{custom_llm_provider}/{model}",
                         kwargs=kwargs,
-                        cached_result=cached_result,
+                        cached_result=converted_result,
                         is_async=False,
                         custom_llm_provider=custom_llm_provider,
                     )
 
                     if not is_internal_call.get() and not _should_defer_streaming_cache_hit_callbacks(
-                        cached_result=cached_result
+                        cached_result=converted_result
                     ):
                         logging_obj.handle_sync_cache_hit_success_callbacks(
-                            result=cached_result,
+                            result=converted_result,
                             start_time=start_time,
                             end_time=end_time,
                             cache_hit=cache_hit,
@@ -527,8 +532,8 @@ class LLMCachingHandler:
                         or self.request_kwargs.get("cache_key")
                         or litellm.cache.get_cache_key(**self.request_kwargs)
                     )
-                    _set_cached_hidden_param(cached_result, "cache_key", cache_key)
-                    return CachingHandlerResponse(cached_result=cached_result)
+                    _set_cached_hidden_param(converted_result, "cache_key", cache_key)
+                    return CachingHandlerResponse(cached_result=converted_result)
         return CachingHandlerResponse(cached_result=cached_result)
 
     _sync_get_cache = sync_get_cache
@@ -950,22 +955,14 @@ class LLMCachingHandler:
 
     def _convert_cached_result_to_model_response(
         self,
-        cached_result: Any,
+        cached_result: object,
         call_type: str,
         kwargs: dict[str, object],
         logging_obj: LiteLLMLoggingObj,
         model: str,
         args: tuple[object, ...],
         custom_llm_provider: str | None = None,
-    ) -> (
-        ModelResponse
-        | TextCompletionResponse
-        | EmbeddingResponse
-        | RerankResponse
-        | TranscriptionResponse
-        | CustomStreamWrapper
-        | None
-    ):
+    ) -> object:
         """
         Internal method to process the cached result
 
@@ -1141,7 +1138,7 @@ class LLMCachingHandler:
         self,
         result: object,
         original_function: Callable,
-        kwargs: dict[str, Any],
+        kwargs: Mapping[str, object],
         args: tuple[object, ...] | None = None,
     ):
         """
@@ -1169,7 +1166,7 @@ class LLMCachingHandler:
             return
         cache: Final = litellm.cache
 
-        new_kwargs: Final = kwargs.copy()
+        new_kwargs: Final = dict(kwargs)
         new_kwargs.update(
             convert_args_to_kwargs(
                 original_function,
@@ -1206,7 +1203,7 @@ class LLMCachingHandler:
                         )
                     )
             else:
-                create_cache_write_task(lambda: cache.async_add_cache(result, **new_kwargs))
+                create_cache_write_task(lambda: cache.async_add_cache(result, **_CacheCallKwargs(**new_kwargs)))
 
     def sync_set_cache(
         self,

litellm/caching/redis_cache.py

@@ -40,7 +40,7 @@ from litellm.constants import (
     REDIS_CIRCUIT_BREAKER_TIMEOUT_MIN_DURATION,
     REDIS_TIMEOUT_LOG_INTERVAL,
 )
-from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs
+from litellm.litellm_core_utils.core_helpers import get_parent_otel_span_from_kwargs, has_attributes
 from litellm.litellm_core_utils.coroutine_checker import coroutine_checker
 from litellm.redis.store_settings import RedisStoreSettings, breaker_settings
 from litellm.types.caching import (
@@ -97,6 +97,22 @@ class _AsyncRedisCommands(Protocol):
     def eval(self, script: str, numkeys: int, *keys_and_args: str | bytes | float) -> Awaitable[object]: ...
 
 
+class _AsyncLuaScript(Protocol):
+    def __call__(
+        self, keys: Sequence[str], args: Sequence[str | bytes | int | float], client: object
+    ) -> Awaitable[object]: ...
+
+
+class _SupportsRegisterScript(Protocol):
+    def register_script(self, script: str, /) -> _AsyncLuaScript: ...
+
+
+class _SupportsScriptLoad(Protocol):
+    def script_load(self, script: str, /) -> object: ...
+
+    def evalsha(self, sha: object, numkeys: int, /, *keys_and_args: str | bytes | float) -> Awaitable[object]: ...
+
+
 _GENERIC_CALLER_MODULES: Final = frozenset(
     {
         __name__,
@@ -844,6 +860,9 @@ class RedisCache(BaseCache):
     def _async_commands(self) -> _AsyncRedisCommands:
         return self.init_async_client()
 
+    def _async_client_object(self) -> object:
+        return self.init_async_client()
+
     def check_and_fix_namespace(self, key: str) -> str:
         """
         Make sure each key starts with the given namespace
@@ -1117,8 +1136,8 @@ class RedisCache(BaseCache):
         Kept separate from async_register_script so each loop caches its own
         executor; see that method for why the binding must be per loop.
         """
-        _redis_client: Final[Any] = self.init_async_client()
-        if hasattr(_redis_client, "register_script"):
+        _redis_client: Final = self._async_client_object()
+        if has_attributes(_redis_client, _SupportsRegisterScript, "register_script"):
             registered_script: Final = _redis_client.register_script(script)
 
             async def standalone_executor(
@@ -1131,7 +1150,7 @@ class RedisCache(BaseCache):
 
             return standalone_executor
 
-        if hasattr(_redis_client, "script_load"):
+        if has_attributes(_redis_client, _SupportsScriptLoad, "script_load"):
             script_sha: Final = _redis_client.script_load(script)
 
             async def cluster_executor(

litellm/caching/redis_semantic_cache.py

@@ -43,6 +43,7 @@ if TYPE_CHECKING:
 _CACHE_RESPONSE_ADAPTER: Final[TypeAdapter[str | bytes | None]] = TypeAdapter(
     str | bytes | None, config=ConfigDict(strict=True, hide_input_in_errors=True)
 )
+_DECODED_CACHE_VALUE: Final = TypeAdapter(object)
 
 
 class RedisSemanticCache(BaseCache):
@@ -407,10 +408,10 @@ class RedisSemanticCache(BaseCache):
 
         # Convert string representation to Python object
         try:
-            return json.loads(response_text)
+            return _DECODED_CACHE_VALUE.validate_python(json.loads(response_text))
         except json.JSONDecodeError:
             try:
-                return ast.literal_eval(response_text)
+                return _DECODED_CACHE_VALUE.validate_python(ast.literal_eval(response_text))
             except (ValueError, SyntaxError) as e:
                 print_verbose(f"Error parsing cached response: {e}")
                 return None

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -39,7 +39,7 @@ from litellm.litellm_core_utils.allowed_tools_tool_choice import (
     parse_chat_allowed_tools_tool_choice,
     responses_allowed_tools_tool_choice,
 )
-from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, iter_items
 from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
 from litellm.litellm_core_utils.prompt_templates.prompt_cache_breakpoint_utils import (
     prompt_cache_breakpoint_with_readable_mode,
@@ -54,6 +54,7 @@ from litellm.llms.base_llm.bridges.completion_transformation import (
     CompletionTransformationBridge,
 )
 from litellm.llms.custom_httpx.sse_lines import split_sse_lines
+from litellm.responses.additional_tools import additional_tools_in_input
 from litellm.responses.litellm_completion_transformation.tool_search import is_anthropic_tool_search_chat_tool
 from litellm.responses.sse_output_recovery import (
     parse_sse_json_chunk,
@@ -348,6 +349,9 @@ _TOOL_SEARCH_REPLAY_ITEM_TYPES: Final = frozenset({"reasoning", *_TOOL_SEARCH_IT
 _TOOL_SEARCH_ITEMS_KEY: Final = "tool_search_items"
 
 
+_DECODED_JSON: Final = TypeAdapter(object)
+
+
 def _json_field(value: object, key: str) -> object:
     return cast(Mapping[str, object], value).get(key) if isinstance(value, dict) else None  # cast-ok: untyped JSON
 
@@ -426,6 +430,17 @@ def replayed_tool_search_tool_names(request: object) -> tuple[str, ...]:
     return tuple(chain.from_iterable(_tool_definition_name(tool) for tool in replayed_tool_search_tools(request)))
 
 
+def _top_level_namespaces(request: object) -> Iterator[object]:
+    tools: Final = _json_list(_json_field(request, "tools")) or ()
+    return (tool for tool in tools if _json_field(tool, "type") == "namespace")
+
+
+def nested_tool_member_names(request: object) -> tuple[str, ...]:
+    tools: Final = (*_top_level_namespaces(request), *additional_tools_in_input(_json_field(request, "input")))
+    members: Final = chain.from_iterable(_namespace_members(tool) for tool in tools)
+    return tuple(chain.from_iterable(_tool_definition_name(member) for member in members))
+
+
 def _loads_any_tool(message: object, names: frozenset[str]) -> bool:
     loaded_names: Final = chain.from_iterable(_tool_definition_name(tool) for tool in _message_loaded_tools(message))
     return any(name in names for name in loaded_names)
@@ -845,8 +860,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
 
         # Handle message items with output_text content
         if item_type == "message":
-            content_list: Final = item.get("content", [])
-            for content_item in content_list:
+            content_list: Final = _DECODED_JSON.validate_python(item.get("content", []))
+            for content_item in iter_items(content_list):
                 if isinstance(content_item, dict):
                     content_type = content_item.get("type")
                     if content_type == "output_text":

litellm/cost_calculator.py

@@ -8,7 +8,7 @@ from datetime import datetime
 from functools import lru_cache
 from itertools import accumulate
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, cast
 
 from httpx import Response
 from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
@@ -24,7 +24,12 @@ from litellm.constants import (
     REALTIME_METERED_USAGE_KEY,
 )
 from litellm.litellm_core_utils.audio_utils.utils import normalized_audio_duration_seconds
-from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, billed_usage_cost
+from litellm.litellm_core_utils.core_helpers import (
+    as_str_mapping,
+    attribute_or_none,
+    billed_usage_cost,
+    require_attributes,
+)
 from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
 from litellm.litellm_core_utils.decisions_pricing import (
     decisions_pricing_fields,
@@ -1188,6 +1193,10 @@ def _model_contains_known_llm_provider(model: str) -> bool:
     return _provider_prefix in LlmProvidersSet
 
 
+class _SupportsGet(Protocol):
+    def get(self, key: str, /) -> object: ...
+
+
 def _get_response_model(completion_response: object) -> str | None:
     """
     Extract the model name from a completion response object.
@@ -1219,7 +1228,7 @@ _GEMINI_TRAFFIC_TYPE_TO_SERVICE_TIER: Final[dict] = {
 }
 
 
-def _map_traffic_type_to_service_tier(traffic_type: str | None) -> str | None:
+def _map_traffic_type_to_service_tier(traffic_type: object) -> str | None:
     """
     Map a Gemini usageMetadata.trafficType value to a LiteLLM service_tier string.
 
@@ -1273,11 +1282,13 @@ def _served_service_tier(completion_response: object, usage_object: Usage | None
     usage_tier: Final = _extract_service_tier(usage_object)
     if isinstance(usage_tier, str):
         return usage_tier
-    hidden_params: Final = getattr(completion_response, "_hidden_params", None)
+    hidden_params: Final[object] = getattr(completion_response, "_hidden_params", None)
     if hidden_params is None:
         return None
-    provider_specific: Final = hidden_params.get("provider_specific_fields") or {}
-    raw_traffic_type: Final = provider_specific.get("traffic_type")
+    provider_specific: Final[object] = (
+        require_attributes(hidden_params, _SupportsGet, "get").get("provider_specific_fields") or {}
+    )
+    raw_traffic_type: Final = require_attributes(provider_specific, _SupportsGet, "get").get("traffic_type")
     if not raw_traffic_type:
         return None
     return _map_traffic_type_to_service_tier(raw_traffic_type) or "default"
@@ -1341,8 +1352,16 @@ def get_usage_object(
         return None
 
 
+class _DumpedUsageFields(TypedDict, total=False, extra_items=object):
+    pass
+
+
+def _dumped_usage_fields(usage_model: BaseModel) -> Mapping[str, object]:
+    return usage_model.model_dump()
+
+
 def _usage_from_model(usage_model: BaseModel) -> Usage:
-    return Usage(**usage_model.model_dump())
+    return Usage(**_DumpedUsageFields(**_dumped_usage_fields(usage_model)))
 
 
 def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper:

litellm/exceptions.py

@@ -12,10 +12,11 @@
 import datetime
 import enum
 from collections.abc import Mapping, Sequence
-from typing import Any, Final
+from typing import Any, Final, Protocol, SupportsIndex, SupportsInt, runtime_checkable
 
 import httpx
 import openai
+from typing_extensions import Buffer
 
 import litellm
 from litellm.types.utils import LiteLLMCommonStrings
@@ -1250,6 +1251,19 @@ class BlockedPiiEntityError(Exception):
         super().__init__(self.message)
 
 
+@runtime_checkable
+class _SupportsTrunc(Protocol):
+    def __trunc__(self) -> int: ...
+
+
+def _status_code_int(value: object) -> int:
+    if isinstance(value, (str, Buffer, SupportsInt, SupportsIndex, _SupportsTrunc)):
+        return int(value)
+    raise TypeError(
+        f"int() argument must be a string, a bytes-like object or a real number, not '{type(value).__name__}'"
+    )
+
+
 class MidStreamFallbackError(ServiceUnavailableError):
     def __init__(
         self,
@@ -1265,8 +1279,8 @@ class MidStreamFallbackError(ServiceUnavailableError):
         is_pre_first_chunk: bool = False,
         emitted_disqualifying_content: bool = False,
     ):
-        original_status: Final = getattr(original_exception, "status_code", None)
-        self.status_code = int(original_status) if original_status is not None else 503
+        original_status: Final[object] = getattr(original_exception, "status_code", None)
+        self.status_code = _status_code_int(original_status) if original_status is not None else 503
         self.message = f"litellm.MidStreamFallbackError: {message}"
         self.model = model
         self.llm_provider = llm_provider
@@ -1309,7 +1323,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
         )
 
         # Restore the propagated status and original response/request objects
-        self.status_code = int(original_status) if original_status is not None else 503
+        self.status_code = _status_code_int(original_status) if original_status is not None else 503
         self.response = _saved_response
         self.request = _saved_request
         self.message = _saved_message

litellm/harness/endpoint.py

@@ -14,10 +14,11 @@ import contextlib
 import itertools
 import json
 import logging
+import math
 import secrets
-from collections.abc import AsyncIterable, AsyncIterator, Mapping
+from collections.abc import AsyncIterable, AsyncIterator, Callable, Mapping
 from dataclasses import dataclass
-from types import MappingProxyType, ModuleType
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Protocol
 
 import httpx
@@ -36,6 +37,7 @@ from litellm.constants import (
 from litellm.harness.context import GatewayTarget
 from litellm.harness.errors import HarnessError, HarnessInstallFailed
 from litellm.harness.types import Harness, Usage
+from litellm.litellm_core_utils.core_helpers import has_attributes
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
 from litellm.types.llms.custom_http import httpxSpecialProvider
@@ -45,7 +47,7 @@ if TYPE_CHECKING:
     from starlette.requests import Request
     from starlette.responses import JSONResponse, Response, StreamingResponse
     from starlette.routing import Route
-    from uvicorn import Server
+    from uvicorn import Config, Server
 
 verbose_logger: Final = logging.getLogger("LiteLLM")
 _JSON_VALUE_ADAPTER: Final = TypeAdapter(object, config=ConfigDict(strict=True))
@@ -110,9 +112,22 @@ class _ResponsesModule(Protocol):
     StreamingResponse: type[StreamingResponse]
 
 
+class _UvicornModule(Protocol):
+    Config: type[Config]
+    Server: type[Server]
+
+
+class _CapturesSignals(Protocol):
+    capture_signals: Callable[[], contextlib.AbstractContextManager[None]]
+
+
+class _InstallsSignalHandlers(Protocol):
+    install_signal_handlers: Callable[[], None]
+
+
 @dataclass(frozen=True)
 class _ServerDeps:
-    uvicorn: ModuleType
+    uvicorn: _UvicornModule
     applications: _ApplicationsModule
     routing: _RoutingModule
     responses: _ResponsesModule
@@ -438,6 +453,14 @@ def _noop() -> None:
     return None
 
 
+def _signal_capture(server: object) -> _CapturesSignals | None:
+    return server if has_attributes(server, _CapturesSignals, "capture_signals") else None
+
+
+def _signal_handler_installer(server: object) -> _InstallsSignalHandlers | None:
+    return server if has_attributes(server, _InstallsSignalHandlers, "install_signal_handlers") else None
+
+
 class ModelEndpoint:
     """Local HTTP endpoint for one harness session. Use as an async context manager."""
 
@@ -521,7 +544,7 @@ class ModelEndpoint:
             await asyncio.sleep(DEFAULT_POLLING_INTERVAL)
 
     def _build_server(self, deps: _ServerDeps) -> Server:
-        config = deps.uvicorn.Config(
+        config: Final = deps.uvicorn.Config(
             self._build_app(deps),
             host=HARNESS_ENDPOINT_HOST,
             port=0,
@@ -529,14 +552,16 @@ class ModelEndpoint:
             log_level="warning",
             access_log=False,
             lifespan="off",
-            timeout_graceful_shutdown=HARNESS_PROCESS_KILL_GRACE_SECONDS,
+            timeout_graceful_shutdown=math.ceil(HARNESS_PROCESS_KILL_GRACE_SECONDS),
         )
-        server = deps.uvicorn.Server(config)
+        server: Final = deps.uvicorn.Server(config)
         # Never touch the host process's signal handlers.
-        if hasattr(server, "capture_signals"):
-            server.capture_signals = contextlib.nullcontext
-        if hasattr(server, "install_signal_handlers"):
-            server.install_signal_handlers = _noop
+        signal_capture: Final = _signal_capture(server)
+        if signal_capture is not None:
+            signal_capture.capture_signals = contextlib.nullcontext
+        handler_installer: Final = _signal_handler_installer(server)
+        if handler_installer is not None:
+            handler_installer.install_signal_handlers = _noop
         return server
 
     def _build_app(self, deps: _ServerDeps) -> Starlette:

litellm/images/utils.py

@@ -6,12 +6,25 @@ from pathlib import Path
 from typing import Any, Final, Protocol, cast, get_type_hints, runtime_checkable
 
 import litellm
+from litellm.litellm_core_utils.core_helpers import has_attributes
 from litellm.litellm_core_utils.token_counter import get_image_type
 from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
 from litellm.types.files import FILE_MIME_TYPES, FileType
 from litellm.types.images.main import ImageEditOptionalRequestParams
 
 
+class _ReadsBytes(Protocol):
+    def read(self, size: int, /) -> bytes: ...
+
+
+class _Tells(Protocol):
+    def tell(self) -> int: ...
+
+
+class _Seeks(Protocol):
+    def seek(self, offset: int, /) -> object: ...
+
+
 class ImageEditRequestUtils:
     @staticmethod
     def get_optional_params_image_edit(
@@ -114,12 +127,12 @@ class ImageEditRequestUtils:
                 bytes_data = image_data[:100]
             else:
                 # For other types, try to read if possible
-                if hasattr(image_data, "read"):
-                    current_pos = getattr(image_data, "tell", lambda: 0)()
-                    if hasattr(image_data, "seek"):
+                if has_attributes(image_data, _ReadsBytes, "read"):
+                    current_pos = image_data.tell() if has_attributes(image_data, _Tells, "tell") else 0
+                    if has_attributes(image_data, _Seeks, "seek"):
                         image_data.seek(0)
                     bytes_data = image_data.read(100)
-                    if hasattr(image_data, "seek"):
+                    if has_attributes(image_data, _Seeks, "seek"):
                         image_data.seek(current_pos)
                 else:
                     return FILE_MIME_TYPES[FileType.PNG]  # Default fallback

litellm/integrations/SlackAlerting/user_spend_alerts.py

@@ -53,11 +53,13 @@ async def fetch_user_spend_rows(
     today: datetime.date,
     baseline_days: int,
 ) -> tuple[UserSpendRow, ...]:
+    from litellm.proxy.utils import query_raw_rows
+
     today_str: Final = today.strftime("%Y-%m-%d")
     month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d")
     baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d")
     return USER_SPEND_ROWS_ADAPTER.validate_python(
-        await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
+        await query_raw_rows(prisma_client.db, USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
     )
 
 

litellm/integrations/argilla.py

@@ -7,11 +7,12 @@ import json
 import os
 import random
 import types
-from collections.abc import Mapping
-from typing import Final
+from collections.abc import Mapping, Sequence
+from typing import Final, Protocol
 
 import httpx
 from pydantic import BaseModel
+from typing_extensions import ReadOnly, TypedDict
 
 import litellm
 from litellm._logging import verbose_logger
@@ -29,6 +30,22 @@ from litellm.types.integrations.argilla import (
 from litellm.types.utils import StandardLoggingPayload
 
 
+class _ArgillaDataset(TypedDict):
+    id: ReadOnly[str]
+
+
+class _ArgillaDatasetsBody(TypedDict, total=False):
+    items: ReadOnly[Sequence[_ArgillaDataset]]
+
+
+class _ArgillaDatasetsResponse(Protocol):
+    def json(self) -> _ArgillaDatasetsBody: ...
+
+
+def _datasets_body(response: _ArgillaDatasetsResponse) -> _ArgillaDatasetsBody:
+    return response.json()
+
+
 def is_serializable(value):
     non_serializable_types: Final = (
         types.CoroutineType,
@@ -102,7 +119,7 @@ class ArgillaLogger(CustomBatchLogger):
                 url=f"{_credentials_base_url}/api/v1/me/datasets?name={_credentials_dataset_name}",
                 headers={"X-Argilla-Api-Key": _credentials_api_key},
             )
-            json_response: Final = dataset_response.json()
+            json_response: Final = _datasets_body(dataset_response)
             if (
                 "items" in json_response
                 and isinstance(json_response["items"], list)

litellm/integrations/arize/_utils.py

@@ -1,6 +1,6 @@
 import json
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Final, Protocol
 
 from pydantic import ConfigDict, TypeAdapter
 from typing_extensions import ReadOnly, TypedDict, override
@@ -10,6 +10,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
     BaseLLMObsOTELAttributes,
     safe_set_attribute,
 )
+from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes, subscript
 from litellm.litellm_core_utils.redact_messages import (
     should_redact_message_logging,
 )
@@ -34,32 +35,40 @@ from litellm.integrations._types.open_inference import (
 _JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
 
 
+class _SupportsGetWithDefault(Protocol):
+    def get(self, key: str, default: object = None, /) -> object: ...
+
+
 class ArizeOTELAttributes(BaseLLMObsOTELAttributes):
     @staticmethod
     @override
-    def set_messages(span: "Span", kwargs: dict[str, Any]):
+    def set_messages(span: "Span", kwargs: Mapping[str, object]):
         messages: Final = kwargs.get("messages")
 
         # for /chat/completions
         # https://docs.arize.com/arize/large-language-models/tracing/semantic-conventions
         if messages:
-            last_message: Final = messages[-1]
+            last_message: Final = subscript(messages, -1)
             safe_set_attribute(
                 span,
                 SpanAttributes.INPUT_VALUE,
-                last_message.get("content", ""),
+                require_attributes(last_message, _SupportsGetWithDefault, "get").get("content", ""),
             )
 
             # LLM_INPUT_MESSAGES shows up under `input_messages` tab on the span page.
-            for idx, msg in enumerate(messages):
+            for idx, msg in enumerate(iter_items(messages)):
                 prefix = f"{SpanAttributes.LLM_INPUT_MESSAGES}.{idx}"
                 # Set the role per message.
-                safe_set_attribute(span, f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", msg.get("role"))
+                safe_set_attribute(
+                    span,
+                    f"{prefix}.{MessageAttributes.MESSAGE_ROLE}",
+                    require_attributes(msg, _SupportsGetWithDefault, "get").get("role"),
+                )
                 # Set the content per message.
                 safe_set_attribute(
                     span,
                     f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}",
-                    msg.get("content", ""),
+                    require_attributes(msg, _SupportsGetWithDefault, "get").get("content", ""),
                 )
 
                 # Additive: emit structured tool_calls / multimodal content

litellm/integrations/azure_sentinel/azure_sentinel.py

@@ -18,10 +18,11 @@ import time
 import traceback
 from collections.abc import Mapping, Sequence
 from types import MappingProxyType
-from typing import Final, TypeVar
+from typing import Final, Protocol, TypeVar
 from urllib.parse import urlparse
 
 import httpx
+from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_logger
 from litellm.integrations.batch_utils import (
@@ -52,6 +53,19 @@ MONITOR_SCOPE_BY_AUTHORITY_HOST: Final[Mapping[str, str]] = MappingProxyType(
 )
 
 
+class _OAuthTokenBody(TypedDict, total=False):
+    access_token: ReadOnly[str]
+    expires_in: ReadOnly[int]
+
+
+class _OAuthTokenResponse(Protocol):
+    def json(self) -> _OAuthTokenBody: ...
+
+
+def _oauth_token_body(response: _OAuthTokenResponse) -> _OAuthTokenBody:
+    return response.json()
+
+
 class AzureSentinelLogger(CustomBatchLogger):
     """
     Logger that sends LiteLLM logs to Azure Sentinel via Azure Monitor Logs Ingestion API
@@ -226,7 +240,7 @@ class AzureSentinelLogger(CustomBatchLogger):
         if response.status_code != 200:
             raise Exception(f"Failed to get OAuth2 token: {response.status_code} - {response.text}")
 
-        token_response: Final = response.json()
+        token_response: Final = _oauth_token_body(response)
         self.oauth_token = token_response.get("access_token")
         expires_in: Final = token_response.get("expires_in", 3600)
 

litellm/integrations/bitbucket/bitbucket_client.py

@@ -5,7 +5,7 @@ BitBucket API client for fetching .prompt files from BitBucket repositories.
 import base64
 import urllib.parse
 from collections.abc import Mapping
-from typing import Final, TypedDict
+from typing import Final, Protocol, TypedDict
 
 from typing_extensions import NotRequired, ReadOnly
 
@@ -36,6 +36,30 @@ class BitBucketFileMetadata(TypedDict):
     last_modified: ReadOnly[str | None]
 
 
+class _SrcListingResponse(Protocol):
+    def json(self) -> BitBucketSrcListing: ...
+
+
+class _RepositoryInfoResponse(Protocol):
+    def json(self) -> Mapping[str, object]: ...
+
+
+class _BranchListingResponse(Protocol):
+    def json(self) -> BitBucketBranchListing: ...
+
+
+def _src_listing(response: _SrcListingResponse) -> BitBucketSrcListing:
+    return response.json()
+
+
+def _repository_info(response: _RepositoryInfoResponse) -> Mapping[str, object]:
+    return response.json()
+
+
+def _branch_listing(response: _BranchListingResponse) -> BitBucketBranchListing:
+    return response.json()
+
+
 def _sanitize_file_path(file_path: str) -> str:
     """Reject path traversal and URL-encode each path segment."""
     if "#" in file_path or "?" in file_path:
@@ -166,7 +190,7 @@ class BitBucketClient:
             response: Final = self.http_handler.get(url, headers=self.headers)
             response.raise_for_status()
 
-            data: Final[BitBucketSrcListing] = response.json()
+            data: Final = _src_listing(response)
             return [
                 file_path
                 for item in data.get("values", [])
@@ -201,7 +225,7 @@ class BitBucketClient:
         try:
             response: Final = self.http_handler.get(url, headers=self.headers)
             response.raise_for_status()
-            return response.json()
+            return _repository_info(response)
         except Exception as e:
             raise Exception(f"Failed to get repository info: {e}")
 
@@ -231,7 +255,7 @@ class BitBucketClient:
             response: Final = self.http_handler.get(url, headers=self.headers)
             response.raise_for_status()
 
-            data: Final[BitBucketBranchListing] = response.json()
+            data: Final = _branch_listing(response)
             return data.get("values", [])
         except Exception as e:
             raise Exception(f"Failed to get branches: {e}")

litellm/integrations/bitbucket/bitbucket_prompt_manager.py

@@ -4,12 +4,13 @@ Fetches .prompt files from BitBucket repositories and provides team-based access
 """
 
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any, Final, Protocol
 
 from jinja2 import DictLoader, select_autoescape
 from jinja2.sandbox import ImmutableSandboxedEnvironment
 
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
+from litellm.litellm_core_utils.core_helpers import require_attributes
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -26,6 +27,10 @@ from litellm.types.utils import StandardCallbackDynamicParams
 from .bitbucket_client import BitBucketClient
 
 
+class _SupportsGetWithDefault(Protocol):
+    def get(self, key: str, default: object = None, /) -> object: ...
+
+
 class BitBucketPromptTemplate:
     """
     Represents a prompt template loaded from BitBucket.
@@ -35,7 +40,7 @@ class BitBucketPromptTemplate:
         self,
         template_id: str,
         content: str,
-        metadata: dict[str, Any],
+        metadata: Mapping[str, object],
         model: str | None = None,
     ):
         self.template_id = template_id
@@ -44,7 +49,9 @@ class BitBucketPromptTemplate:
         self.model = model or metadata.get("model")
         self.temperature = metadata.get("temperature")
         self.max_tokens = metadata.get("max_tokens")
-        self.input_schema = metadata.get("input", {}).get("schema", {})
+        self.input_schema = require_attributes(metadata.get("input", {}), _SupportsGetWithDefault, "get").get(
+            "schema", {}
+        )
         self.optional_params = {k: v for k, v in metadata.items() if k not in ["model", "input", "content"]}
 
     def __repr__(self):

litellm/integrations/clickhouse/clickhouse_spend_logger.py

@@ -7,10 +7,10 @@ so `response_id` is always the raw provider response id (cache-hit suffix stripp
 
 import json
 import re
-from collections.abc import Iterator, Mapping
+from collections.abc import Iterable, Iterator, Mapping
 from math import isfinite
 from types import MappingProxyType
-from typing import Any, Final
+from typing import Any, Final, Protocol
 
 from pydantic import JsonValue, TypeAdapter, ValidationError
 
@@ -19,6 +19,7 @@ from litellm._logging import verbose_logger
 from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
 from litellm.integrations.clickhouse.context import is_lens_analysis
 from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
+from litellm.litellm_core_utils.core_helpers import require_attributes
 from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
 from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
@@ -40,6 +41,14 @@ _INTERNAL_METADATA_KEYS: Final = frozenset(
 )
 
 
+class _SupportsGet(Protocol):
+    def get(self, key: str, /) -> object: ...
+
+
+class _SupportsItems(Protocol):
+    def items(self) -> Iterable[tuple[object, object]]: ...
+
+
 def _request_metadata_fields(value: object) -> Iterator[tuple[str, JsonValue]]:
     if value is None:
         return
@@ -85,22 +94,22 @@ def _json_mapping(value: Mapping[str, object]) -> str:
     return _json(dict(value))
 
 
-def _find_traceparent(metadata: Mapping[str, Any], kwargs: Mapping[str, Any]) -> tuple[str, str]:
+def _find_traceparent(metadata: Mapping[str, object], kwargs: Mapping[str, object]) -> tuple[str, str]:
     custom_headers = metadata.get("requester_custom_headers") or MappingProxyType({})
-    proxy_request = (kwargs.get("litellm_params") or MappingProxyType({})).get(
+    proxy_request = require_attributes(kwargs.get("litellm_params") or MappingProxyType({}), _SupportsGet, "get").get(
         "proxy_server_request"
     ) or MappingProxyType({})
-    request_headers = proxy_request.get("headers") or MappingProxyType({})
+    request_headers = require_attributes(proxy_request, _SupportsGet, "get").get("headers") or MappingProxyType({})
     for headers in (custom_headers, request_headers):
-        for name, value in headers.items():
+        for name, value in require_attributes(headers, _SupportsItems, "items").items():
             if str(name).lower() == "traceparent":
                 return parse_traceparent(value)
     return "", ""
 
 
-def _cache_tokens(usage: Mapping[str, Any]) -> tuple[int, int]:
+def _cache_tokens(usage: Mapping[str, object]) -> tuple[int, int]:
     """(cache_read, cache_write) from a Usage dict: OpenAI prompt_tokens_details first, Anthropic fields as fallback."""
-    details = usage.get("prompt_tokens_details") or MappingProxyType({})
+    details = require_attributes(usage.get("prompt_tokens_details") or MappingProxyType({}), _SupportsGet, "get")
     cache_read = _int(details.get("cached_tokens")) or _int(usage.get("cache_read_input_tokens"))
     cache_write = (
         _int(details.get("cache_write_tokens"))
@@ -116,10 +125,17 @@ def _request_tags(value: object) -> list[str]:
     return [str(tag) for tag in value]
 
 
-def _session_id(payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, Any]) -> str:
+def _session_id(payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, object]) -> str:
     """Mirrors proxy `_get_session_id_for_spend_log`: explicit session id, else the payload trace id."""
-    request_metadata = (kwargs.get("litellm_params") or MappingProxyType({})).get("metadata") or MappingProxyType({})
-    return str(payload.get("session_id") or request_metadata.get("session_id") or payload.get("trace_id") or "")
+    request_metadata = require_attributes(
+        kwargs.get("litellm_params") or MappingProxyType({}), _SupportsGet, "get"
+    ).get("metadata") or MappingProxyType({})
+    return str(
+        payload.get("session_id")
+        or require_attributes(request_metadata, _SupportsGet, "get").get("session_id")
+        or payload.get("trace_id")
+        or ""
+    )
 
 
 def _is_trace_ingest(payload: StandardLoggingPayload) -> bool:
@@ -128,7 +144,7 @@ def _is_trace_ingest(payload: StandardLoggingPayload) -> bool:
 
 
 def spend_log_row_from_payload(
-    payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, Any]
+    payload: StandardLoggingPayload | SpendLogPayload, kwargs: Mapping[str, object]
 ) -> SpendLogRecord:
     metadata: Mapping[str, Any] = payload.get("metadata") or MappingProxyType({})
     hidden_params: Mapping[str, Any] = payload.get("hidden_params") or MappingProxyType({})

litellm/integrations/custom_guardrail.py

@@ -1,16 +1,17 @@
 import contextvars
 import copy
 import hashlib
+import json
 import os
 import secrets
 from collections.abc import Iterable, Mapping, Sequence
 from datetime import datetime
 from itertools import chain
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, Protocol, cast, get_args
 
 import httpx
-from pydantic import TypeAdapter
+from pydantic import BaseModel, TypeAdapter
 
 from litellm._internal_context import with_service_target
 from litellm._logging import verbose_logger
@@ -20,6 +21,7 @@ from litellm.litellm_core_utils.core_helpers import (
     get_metadata_variable_name_from_kwargs,
     get_or_create_metadata_bucket,
     redact_nested_match_and_regex_keys,
+    require_attributes,
 )
 from litellm.secret_managers.main import str_to_bool
 from litellm.types.guardrails import (
@@ -73,6 +75,7 @@ GUARDRAIL_SESSIONS_TARGET: Final = "guardrail_sessions"
 _PRE_CALL_EXECUTED_TOKEN: Final = secrets.token_hex(16)
 _PRE_CALL_RAISED_TOKEN: Final = secrets.token_hex(16)
 _DEPLOYMENT_HOOK_RAN_TOKEN: Final = secrets.token_hex(16)
+_POST_CALL_CHECKED_TOKEN: Final = secrets.token_hex(16)
 
 _GUARDRAIL_BLOCK_STATUS_CODES: Final = frozenset({400, 403, 422})
 
@@ -167,18 +170,22 @@ def _strict_guardrail_modes_enabled() -> bool:
     return True if parsed is None else parsed
 
 
-def get_session_id_from_request_data(request_data: dict[str, Any]) -> str | None:
+class _SupportsGet(Protocol):
+    def get(self, key: str, /) -> object: ...
+
+
+def get_session_id_from_request_data(request_data: Mapping[str, object]) -> str | None:
     """Extract session_id from request data (litellm_session_id or metadata)."""
     session_id = request_data.get("litellm_session_id")
     if session_id:
         return str(session_id)
 
-    metadata: Final = request_data.get("metadata") or {}
+    metadata: Final = require_attributes(request_data.get("metadata") or {}, _SupportsGet, "get")
     session_id = metadata.get("session_id")
     if session_id:
         return str(session_id)
 
-    litellm_metadata: Final = request_data.get("litellm_metadata") or {}
+    litellm_metadata: Final = require_attributes(request_data.get("litellm_metadata") or {}, _SupportsGet, "get")
     session_id = litellm_metadata.get("session_id")
     if session_id:
         return str(session_id)
@@ -281,6 +288,95 @@ def _request_carries_marker(data: Mapping[str, object], marker: str | None) -> b
     return False
 
 
+def _record_guardrail_marker(
+    data: dict[str, object],  # mutable-ok: the request payload the marker is recorded on
+    marker: str | None,
+) -> None:
+    if marker is None:
+        return
+    for meta_key in ("metadata", "litellm_metadata"):
+        meta = data.get(meta_key)
+        if isinstance(meta, dict):
+            executed = meta.get(PRE_CALL_EXECUTED_GUARDRAILS_KEY)
+            if isinstance(executed, list):
+                if marker not in executed:
+                    executed.append(marker)
+            else:
+                meta[PRE_CALL_EXECUTED_GUARDRAILS_KEY] = [marker]
+            return
+    data["metadata"] = {PRE_CALL_EXECUTED_GUARDRAILS_KEY: [marker]}
+
+
+_RESPONSE_METADATA_KEYS: Final = frozenset(
+    {
+        "id",
+        "created",
+        "created_at",
+        "model",
+        "usage",
+        "usage_info",
+        "system_fingerprint",
+        "_hidden_params",
+        "_response_ms",
+    }
+)
+
+
+def _response_fingerprint(response: object) -> str | None:
+    """Digest of everything but metadata set between layers, so an unknown field counts as content."""
+    try:
+        fields: Final[Mapping[str, object]] = (
+            response.model_dump(mode="json")
+            if isinstance(response, BaseModel)
+            else _REQUEST_FIELDS.validate_python(response)
+        )
+        content: Final = {key: value for key, value in fields.items() if key not in _RESPONSE_METADATA_KEYS}
+        if not content:
+            return None
+        return hashlib.sha256(
+            json.dumps(content, sort_keys=True, separators=(",", ":"), allow_nan=False).encode()
+        ).hexdigest()
+    except (TypeError, ValueError):
+        return None
+
+
+_RESPONSE_GUARDRAIL_CHECKS_KEY: Final = "guardrail_checks"
+
+
+def _str_keyed_dict(value: object) -> Mapping[str, object] | None:
+    if not isinstance(value, dict):
+        return None
+    return cast("Mapping[str, object]", value)  # cast-ok: response bodies and _hidden_params are keyed by field name
+
+
+def _response_hidden_params(response: object) -> Mapping[str, object] | None:
+    as_dict: Final = _str_keyed_dict(response)
+    return _str_keyed_dict(
+        as_dict.get("_hidden_params") if as_dict is not None else getattr(response, "_hidden_params", None)
+    )
+
+
+def _write_response_marker(response: object, marker: str | None) -> None:
+    """Record a check on the response itself, so it survives the router copying request metadata per hop."""
+    if marker is None:
+        return
+    hidden: Final = _response_hidden_params(response)
+    if not isinstance(hidden, dict):
+        return
+    hidden[_RESPONSE_GUARDRAIL_CHECKS_KEY] = tuple(
+        _str_collection(hidden.get(_RESPONSE_GUARDRAIL_CHECKS_KEY)) | {marker}
+    )
+
+
+def _response_carries_marker(response: object, marker: str | None) -> bool:
+    hidden: Final = _response_hidden_params(response)
+    return (
+        marker is not None
+        and hidden is not None
+        and marker in _str_collection(hidden.get(_RESPONSE_GUARDRAIL_CHECKS_KEY))
+    )
+
+
 _REQUEST_FIELDS: Final = TypeAdapter(Mapping[str, object])
 
 
@@ -291,6 +387,7 @@ def _guardrails_listed_in(metadata: object) -> list[str | dict[str, DynamicGuard
 
 
 class CustomGuardrail(CustomLogger):
+    mcp_block_behavior: Literal["tool_result", "raise"] | None = None
     # If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path.
     use_native_during_call_hook: ClassVar[bool] = False
 
@@ -912,20 +1009,7 @@ class CustomGuardrail(CustomLogger):
         With ``raised``, records instead that the hook raised as a policy pipeline
         step the pipeline then moved past. The deployment-level hook still runs it.
         """
-        marker: Final = self._pre_call_marker(raised=raised)
-        if marker is None:
-            return
-        for meta_key in ("metadata", "litellm_metadata"):
-            meta = data.get(meta_key)
-            if isinstance(meta, dict):
-                executed = meta.get(PRE_CALL_EXECUTED_GUARDRAILS_KEY)
-                if isinstance(executed, list):
-                    if marker not in executed:
-                        executed.append(marker)
-                else:
-                    meta[PRE_CALL_EXECUTED_GUARDRAILS_KEY] = [marker]
-                return
-        data["metadata"] = {PRE_CALL_EXECUTED_GUARDRAILS_KEY: [marker]}
+        _record_guardrail_marker(data, self._pre_call_marker(raised=raised))
 
     def pre_call_hook_ran_or_raised(self, data: Mapping[str, object]) -> bool:
         """Whether this guardrail's ``async_pre_call_hook`` ran for this request, whatever it decided."""
@@ -1123,10 +1207,26 @@ class CustomGuardrail(CustomLogger):
                 cast("dict[str, object]", request_data)  # cast-ok: proxy request payloads are str-keyed
             )
 
-        if not self._is_valid_response_type(result):
-            return None
+        returned: Final = result if self._is_valid_response_type(result) else None
+        if target is self:
+            return returned
+        checked: Final = returned if returned is not None else response
+        marker: Final = self._post_call_check_marker(checked)
+        _record_guardrail_marker(
+            cast("dict[str, object]", request_data),  # cast-ok: proxy request payloads are str-keyed
+            marker,
+        )
+        _write_response_marker(checked, marker)
+        return returned
 
-        return result
+    def _post_call_check_marker(self, response: object) -> str | None:
+        fingerprint: Final = _response_fingerprint(response)
+        return self._marker(f"{_POST_CALL_CHECKED_TOKEN}:{fingerprint}") if fingerprint is not None else None
+
+    def post_call_already_checked(self, data: Mapping[str, object], response: object) -> bool:
+        """Whether a deployment-level hook of this request already ran this guardrail on this exact response."""
+        marker: Final = self._post_call_check_marker(response)
+        return _request_carries_marker(data, marker) or _response_carries_marker(response, marker)
 
     async def async_logging_hook(
         self,

litellm/integrations/email_alerting.py

@@ -3,17 +3,29 @@ Functions for sending Email Alerts
 """
 
 import os
-from typing import Final
+from collections.abc import Awaitable, Mapping, Sequence
+from typing import TYPE_CHECKING, Final
 
 from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm.repositories.team_repository import TeamRepository
 from litellm.types.integrations.slack_alerting import WebhookEvent
 
+if TYPE_CHECKING:
+    from litellm.proxy.utils import PrismaClient
+
 # we use this for the email header, please send a test email if you change this. verify it looks good on email
 LITELLM_LOGO_URL: Final = "https://litellm-listing.s3.amazonaws.com/litellm_logo.png"
 LITELLM_SUPPORT_CONTACT: Final = "support@berri.ai"
 
 
+def _query_raw_rows(
+    client: "PrismaClient", query: str, *args: object
+) -> Awaitable[Sequence[Mapping[str, object]] | None]:
+    from litellm.proxy.utils import query_raw_rows
+
+    return query_raw_rows(client.db, query, *args)
+
+
 async def get_all_team_member_emails(team_id: str | None = None) -> list:
     verbose_logger.debug("Email Alerting: Getting all team members for team_id=%s", team_id)
     if team_id is None:
@@ -51,17 +63,18 @@ async def get_all_team_member_emails(team_id: str | None = None) -> list:
         WHERE user_id = ANY($1::TEXT[]);
     """
 
-    _result: Final = await prisma_client.db.query_raw(sql_query, _team_member_user_ids)
+    _result: Final = await _query_raw_rows(prisma_client, sql_query, _team_member_user_ids)
 
     verbose_logger.debug("Email Alerting: Got all Emails for team, emails=%s", _result)
 
     if _result is None:
         return []
 
-    emails: Final = []
-    for user in _result:
-        if user and isinstance(user, dict) and user.get("user_email", None) is not None:
-            emails.append(user.get("user_email"))
+    emails: Final = [
+        user.get("user_email")
+        for user in _result
+        if user and isinstance(user, dict) and user.get("user_email", None) is not None
+    ]
     return emails
 
 

litellm/integrations/focus/database.py

@@ -123,7 +123,9 @@ class FocusLiteLLMDatabase:
         ORDER BY ordinal_position;
         """
         try:
-            columns_response: Final = await client.db.query_raw(info_query)
+            from litellm.proxy.utils import query_raw_rows
+
+            columns_response: Final = await query_raw_rows(client.db, info_query)
             return {"columns": columns_response, "table_name": "LiteLLM_DailyUserSpend"}
         except Exception as exc:
             raise RuntimeError(f"Error getting table info: {exc}") from exc

litellm/integrations/galileo.py

@@ -15,6 +15,7 @@ from typing_extensions import ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.core_helpers import require_attributes, subscript
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     convert_content_list_to_str,
     get_content_from_model_response,
@@ -60,6 +61,10 @@ class _JsonResponse(Protocol):
     def json(self) -> object: ...
 
 
+class _SupportsGet(Protocol):
+    def get(self, key: str, /) -> object: ...
+
+
 def _login_access_token(response: _GalileoLoginResponse) -> str:
     """Read the bearer token out of a Galileo login response body."""
     return response.json()["access_token"]
@@ -463,13 +468,13 @@ class GalileoObserve(CustomLogger):
             pass
 
     @staticmethod
-    def _build_prompt(kwargs: Mapping[str, Any]) -> dict[str, object]:
-        optional_params: Final[Mapping[str, object]] = kwargs.get("optional_params", {}) or {}
+    def _build_prompt(kwargs: Mapping[str, object]) -> dict[str, object]:
+        optional_params: Final = require_attributes(kwargs.get("optional_params", {}) or {}, _SupportsGet, "get")
         prompt: Final[dict[str, object]] = {"messages": kwargs.get("messages")}
         if optional_params.get("functions") is not None:
-            prompt["functions"] = optional_params["functions"]
+            prompt["functions"] = subscript(optional_params, "functions")
         if optional_params.get("tools") is not None:
-            prompt["tools"] = optional_params["tools"]
+            prompt["tools"] = subscript(optional_params, "tools")
         return prompt
 
     @staticmethod

litellm/integrations/gcs_bucket/gcs_bucket.py

@@ -5,7 +5,7 @@ import os
 import time
 from collections.abc import Mapping
 from datetime import datetime, timedelta, timezone
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any, Final, Protocol
 
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
@@ -16,7 +16,7 @@ from litellm.integrations.gcs_bucket.gcs_bucket_base import GCSBucketBase
 from litellm.litellm_core_utils.cloud_storage_security import (
     sanitize_cloud_object_component,
 )
-from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, require_attributes
 from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.types.integrations.base_health_check import IntegrationHealthCheckStatus
 from litellm.types.integrations.gcs_bucket import *
@@ -28,6 +28,10 @@ else:
     VertexBase = Any
 
 
+class _SupportsGetWithDefault(Protocol):
+    def get(self, key: str, default: str, /) -> object: ...
+
+
 class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
     def __init__(self, bucket_name: str | None = None) -> None:
         from litellm.proxy.proxy_server import premium_user
@@ -298,7 +302,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
             await self._send_individual_logs(items_to_process)
 
     def _get_object_name(
-        self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: Any
+        self, kwargs: Mapping[str, object], logging_payload: StandardLoggingPayload, response_obj: object
     ) -> str:
         """
         Get the object name to use for the current payload
@@ -313,7 +317,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
             return f"{current_date}/custom-{uuid.uuid4().hex}-{safe_log_id}"
         if logging_payload.get("error_str", None) is not None:
             return self._generate_failure_object_name(request_date_str=current_date)
-        raw_response_id: Final[object] = response_obj.get("id", "")
+        raw_response_id: Final[object] = require_attributes(response_obj, _SupportsGetWithDefault, "get").get("id", "")
         response_id: Final = raw_response_id if isinstance(raw_response_id, str) else ""
         return self._generate_success_object_name(
             request_date_str=current_date,
@@ -360,7 +364,7 @@ class GCSBucketLogger(GCSBucketBase, AdditionalLoggingUtils):
     def _generate_success_object_name(
         self,
         request_date_str: str,
-        response_id: str,
+        response_id: object,
     ) -> str:
         return f"{request_date_str}/{response_id}"
 

litellm/integrations/generic_prompt_management/generic_prompt_manager.py

@@ -132,7 +132,7 @@ class GenericPromptManager(CustomPromptManagement):
 
     async def async_fetch_prompt_from_api(
         self, prompt_id: str | None, prompt_spec: PromptSpec | None
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Fetch a prompt from the API asynchronously.
         """

litellm/integrations/humanloop.py

@@ -7,10 +7,12 @@ https://humanloop.com/
 from typing import Final, cast
 
 import httpx
+from pydantic import TypeAdapter
 from typing_extensions import TypedDict
 
 import litellm
 from litellm.caching import DualCache
+from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping, decoded_json_object, subscript
 from litellm.llms.custom_httpx.http_handler import get_httpx_client
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
@@ -19,6 +21,8 @@ from litellm.types.utils import StandardCallbackDynamicParams
 
 from .custom_logger import CustomLogger
 
+_DECODED_JSON: Final = TypeAdapter(object)
+
 
 class PromptManagementClient(TypedDict):
     prompt_id: str
@@ -79,18 +83,20 @@ class HumanLoopPromptManager(DualCache):
             raise Exception(f"Error getting prompt from Humanloop: {e.response.text}")
 
         json_response: Final = response.json()
-        template_message: Final = json_response["template"]
-        if isinstance(template_message, dict):
-            template_messages = [template_message]
-        elif isinstance(template_message, list):
-            template_messages = template_message
+        decoded_response: Final = _DECODED_JSON.validate_python(json_response)
+        template_message: Final = subscript(decoded_response, "template")
+        template_mapping: Final = as_str_mapping(template_message)
+        template_list: Final = as_object_list(template_message)
+        if template_mapping is not None:
+            template_messages = [template_mapping]
+        elif template_list is not None:
+            template_messages = template_list
         else:
             raise ValueError(f"Invalid template message type: {type(template_message)}")
         template_model: Final = json_response["model"]
-        optional_params: Final = {}
-        for k, v in json_response.items():
-            if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS:
-                optional_params[k] = v
+        optional_params: Final = {
+            k: v for k, v in decoded_json_object(decoded_response).items() if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS
+        }
         return PromptManagementClient(
             prompt_id=humanloop_prompt_id,
             prompt_template=cast(list[AllMessageValues], template_messages),

litellm/integrations/lago.py

@@ -3,7 +3,7 @@
 
 import json
 import os
-from typing import Final, Literal
+from typing import Final, Literal, Protocol
 
 import httpx
 
@@ -11,6 +11,7 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.core_helpers import require_attributes
 from litellm.llms.custom_httpx.http_handler import (
     HTTPHandler,
     get_async_httpx_client,
@@ -18,6 +19,10 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 
 
+class _SupportsGet(Protocol):
+    def get(self, key: str, default: object = None, /) -> object: ...
+
+
 def get_utc_datetime():
     import datetime as dt
     from datetime import datetime
@@ -70,10 +75,11 @@ class LagoLogger(CustomLogger):
         if (
             isinstance(response_obj, litellm.ModelResponse) or isinstance(response_obj, litellm.EmbeddingResponse)
         ) and hasattr(response_obj, "usage"):
+            response_usage: Final = require_attributes(response_obj["usage"], _SupportsGet, "get")
             usage = {
-                "prompt_tokens": response_obj["usage"].get("prompt_tokens", 0),
-                "completion_tokens": response_obj["usage"].get("completion_tokens", 0),
-                "total_tokens": response_obj["usage"].get("total_tokens"),
+                "prompt_tokens": response_usage.get("prompt_tokens", 0),
+                "completion_tokens": response_usage.get("completion_tokens", 0),
+                "total_tokens": response_usage.get("total_tokens"),
             }
 
         litellm_params: Final = kwargs.get("litellm_params", {}) or {}

litellm/integrations/langfuse/langfuse_otel.py

@@ -7,6 +7,7 @@ from itertools import chain
 from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, runtime_checkable
 
 from opentelemetry import trace
+from pydantic import TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
@@ -33,6 +34,8 @@ from litellm.types.integrations.langfuse_otel import (
 )
 from litellm.types.utils import StandardCallbackDynamicParams
 
+_DECODED_JSON: Final = TypeAdapter(object)
+
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
 
@@ -643,7 +646,7 @@ def _tool_call_arguments(arguments: object) -> object:
     if not isinstance(arguments, str):
         return arguments
     try:
-        return json.loads(arguments)
+        return _DECODED_JSON.validate_python(json.loads(arguments))
     except json.JSONDecodeError:
         return {}
 

litellm/integrations/langfuse/langfuse_otel_attributes.py

@@ -7,7 +7,7 @@ Relevant Issue: https://github.com/BerriAI/litellm/issues/13764
 import json
 from collections.abc import Mapping
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Final, Protocol
 
 from pydantic import BaseModel
 from typing_extensions import override
@@ -16,6 +16,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
     BaseLLMObsOTELAttributes,
     safe_set_attribute,
 )
+from litellm.litellm_core_utils.core_helpers import require_attributes
 from litellm.types.llms.openai import HttpxBinaryResponseContent, ResponsesAPIResponse
 from litellm.types.utils import (
     EmbeddingResponse,
@@ -30,6 +31,10 @@ if TYPE_CHECKING:
     from opentelemetry.trace import Span
 
 
+class _SupportsGet(Protocol):
+    def get(self, key: str, /) -> object: ...
+
+
 def serialize_langfuse_datetime(value: datetime) -> str:
     return json.dumps(value.astimezone().isoformat())
 
@@ -87,11 +92,11 @@ def get_output_content_by_type(
 class LangfuseLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
     @staticmethod
     @override
-    def set_messages(span: "Span", kwargs: dict[str, Any]):
+    def set_messages(span: "Span", kwargs: Mapping[str, object]):
         prompt: Final = {"messages": kwargs.get("messages")}
         optional_params: Final = kwargs.get("optional_params", {})
-        functions: Final = optional_params.get("functions")
-        tools: Final = optional_params.get("tools")
+        functions: Final = require_attributes(optional_params, _SupportsGet, "get").get("functions")
+        tools: Final = require_attributes(optional_params, _SupportsGet, "get").get("tools")
         if functions is not None:
             prompt["functions"] = functions
         if tools is not None:

litellm/integrations/mlflow.py

@@ -1,9 +1,11 @@
 import json
 import threading
-from typing import TYPE_CHECKING, Any, Final
+from collections.abc import Sequence
+from typing import TYPE_CHECKING, Any, Final, Protocol
 
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.litellm_core_utils.core_helpers import attribute_or_none, iter_items, require_attributes
 
 if TYPE_CHECKING:
     from litellm.types.utils import StandardLoggingPayload
@@ -11,6 +13,25 @@ else:
     StandardLoggingPayload = Any
 
 
+class _ChoiceWithMessage(Protocol):
+    @property
+    def message(self) -> object: ...
+
+
+class _ExcludeNoneDumpable(Protocol):
+    def model_dump(self, *, exclude_none: bool) -> object: ...
+
+
+def _dumped_choice_message(choice: object) -> object:
+    message: Final = require_attributes(choice, _ChoiceWithMessage, "message").message
+    return require_attributes(message, _ExcludeNoneDumpable, "model_dump").model_dump(exclude_none=True)
+
+
+def _output_messages(response_obj: object) -> Sequence[object]:
+    choices: Final = attribute_or_none(response_obj, "choices") if hasattr(response_obj, "choices") else ()
+    return [_dumped_choice_message(c) for c in iter_items(choices)]
+
+
 class MlflowLogger(CustomLogger):
     def __init__(self):
         from mlflow.tracking import MlflowClient
@@ -62,7 +83,7 @@ class MlflowLogger(CustomLogger):
 
         inputs: Final = self._construct_input(kwargs)
         input_messages: Final = inputs.get("messages", [])
-        output_messages: Final = [c.message.model_dump(exclude_none=True) for c in getattr(response_obj, "choices", [])]
+        output_messages: Final = _output_messages(response_obj)
         if messages := [*input_messages, *output_messages]:
             set_span_chat_messages(span, messages)
         if tools := inputs.get("tools"):

litellm/integrations/openmeter.py

@@ -4,13 +4,13 @@
 import json
 import os
 from collections.abc import Mapping
-from typing import Final
+from typing import Final, Protocol
 
 import httpx
 
 import litellm
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.litellm_core_utils.core_helpers import as_str_mapping
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, require_attributes
 from litellm.litellm_core_utils.internal_call_metadata import logged_event_key
 from litellm.llms.custom_httpx.http_handler import (
     HTTPHandler,
@@ -19,6 +19,10 @@ from litellm.llms.custom_httpx.http_handler import (
 )
 
 
+class _SupportsGet(Protocol):
+    def get(self, key: str, default: object = None, /) -> object: ...
+
+
 def get_utc_datetime():
     import datetime as dt
     from datetime import datetime
@@ -60,10 +64,11 @@ class OpenMeterLogger(CustomLogger):
         if (
             isinstance(response_obj, litellm.ModelResponse) or isinstance(response_obj, litellm.EmbeddingResponse)
         ) and hasattr(response_obj, "usage"):
+            response_usage: Final = require_attributes(response_obj["usage"], _SupportsGet, "get")
             usage = {
-                "prompt_tokens": response_obj["usage"].get("prompt_tokens", 0),
-                "completion_tokens": response_obj["usage"].get("completion_tokens", 0),
-                "total_tokens": response_obj["usage"].get("total_tokens"),
+                "prompt_tokens": response_usage.get("prompt_tokens", 0),
+                "completion_tokens": response_usage.get("completion_tokens", 0),
+                "total_tokens": response_usage.get("total_tokens"),
             }
 
         # OPENMETER_TRUST_REQUEST_USER (default "true"): when set to "false",

litellm/integrations/opentelemetry.py

@@ -6,7 +6,7 @@ from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass, field
 from datetime import datetime
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, cast
+from typing import TYPE_CHECKING, Any, Final, Protocol, cast
 
 from pydantic import ConfigDict, TypeAdapter
 from typing_extensions import ReadOnly, TypedDict
@@ -35,6 +35,7 @@ from litellm.integrations.otel.model.metadata import flatten_metadata
 from litellm.integrations.otel.model.semconv import LiteLLM, Metric
 from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
 from litellm.integrations.otel.routing import routing_decision_attributes
+from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes
 from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.secret_redaction import redact_string
@@ -47,7 +48,6 @@ from litellm.secret_managers.main import get_secret_bool, str_to_bool
 from litellm.types.integrations.otel_span_attributes import SpanAttributes
 from litellm.types.services import ServiceLoggerPayload
 from litellm.types.utils import (
-    ChatCompletionMessageToolCall,
     CostBreakdown,
     Function,
     LLMResponseTypes,
@@ -130,6 +130,22 @@ def _text_as_stored_on_span(span: Span, key: str, text: str) -> str:
 _MAX_DYNAMIC_TRACER_PROVIDERS: Final = 256
 
 
+class _SupportsGet(Protocol):
+    def get(self, key: str, /) -> object: ...
+
+
+def _response_field(response_obj: object, key: str) -> object:
+    if not response_obj:
+        return None
+    return require_attributes(response_obj, _SupportsGet, "get").get(key)
+
+
+def _billed_usage(response_obj: object, call_type: str | None, litellm_params: Mapping[str, object] | None) -> object:
+    if not response_obj or is_unbilled_non_inference_call_from_params(call_type, litellm_params, response_obj):
+        return None
+    return _response_field(response_obj, "usage")
+
+
 class _CacheHitView(TypedDict, total=False):
     cache_hit: ReadOnly[object]
     saved_cache_cost: ReadOnly[object]
@@ -2407,17 +2423,17 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
 
     @staticmethod
     def _tool_calls_kv_pair(
-        tool_calls: list[ChatCompletionMessageToolCall],
+        tool_calls: Iterable[object],
     ) -> dict[str, object]:
         kv_pairs: Final[dict[str, object]] = {}
         for idx, tool_call in enumerate(tool_calls):
-            _function = tool_call.get("function")
+            _function: object = require_attributes(tool_call, _SupportsGet, "get").get("function")
             if not _function:
                 continue
 
             keys = Function.__annotations__.keys()
             for key in keys:
-                _value = _function.get(key)
+                _value = require_attributes(_function, _SupportsGet, "get").get(key)
                 if _value:
                     kv_pairs[f"{SpanAttributes.LLM_COMPLETIONS.value}.{idx}.function_call.{key}"] = _value
 
@@ -2584,7 +2600,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             # but Embeddings and Image-gen responses do not.  Fall back to
             # the litellm call ID so every call type can be correlated
             # across LiteLLM UI, Phoenix traces, and provider logs (Issue #8).
-            response_id = (response_obj.get("id") if response_obj else None) or standard_logging_payload.get("id")
+            response_id = _response_field(response_obj, "id") or standard_logging_payload.get("id")
             if response_id:
                 self.safe_set_attribute(
                     span=span,
@@ -2601,42 +2617,37 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
                 )
 
             # The model used to generate the response.
-            if response_obj and response_obj.get("model"):
+            response_model: Final = _response_field(response_obj, "model")
+            if response_model:
                 self.safe_set_attribute(
                     span=span,
                     key=SpanAttributes.LLM_RESPONSE_MODEL.value,
-                    value=response_obj.get("model"),
+                    value=response_model,
                 )
 
             self._set_service_tier_attributes(span=span, standard_logging_payload=standard_logging_payload)
 
-            usage: Final = (
-                response_obj.get("usage")
-                if response_obj
-                and not is_unbilled_non_inference_call_from_params(
-                    kwargs.get("call_type"), litellm_params, response_obj
-                )
-                else None
-            )
+            usage: Final = _billed_usage(response_obj, kwargs.get("call_type"), litellm_params)
             if usage:
+                usage_fields: Final = require_attributes(usage, _SupportsGet, "get")
                 self.safe_set_attribute(
                     span=span,
                     key=SpanAttributes.GEN_AI_USAGE_TOTAL_TOKENS.value,
-                    value=usage.get("total_tokens"),
+                    value=usage_fields.get("total_tokens"),
                 )
 
                 # The number of tokens used in the LLM response (completion).
                 self.safe_set_attribute(
                     span=span,
                     key=SpanAttributes.GEN_AI_USAGE_OUTPUT_TOKENS.value,
-                    value=usage.get("completion_tokens"),
+                    value=usage_fields.get("completion_tokens"),
                 )
 
                 # The number of tokens used in the LLM prompt.
                 self.safe_set_attribute(
                     span=span,
                     key=SpanAttributes.GEN_AI_USAGE_INPUT_TOKENS.value,
-                    value=usage.get("prompt_tokens"),
+                    value=usage_fields.get("prompt_tokens"),
                 )
 
                 ########################################################################
@@ -2712,7 +2723,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             ########## LLM Response Attributes ##########
             #############################################
             if response_obj is not None:
-                if response_obj.get("choices"):
+                response_view: Final = require_attributes(response_obj, _SupportsGet, "get")
+                if response_view.get("choices"):
                     transformed_choices: Final = self._transform_choices_to_otel_semantic_conventions(
                         response_obj.get("choices")
                     )
@@ -2722,10 +2734,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
                         value=safe_dumps(transformed_choices),
                     )
 
-                    finish_reasons: Final = []
-                    for idx, choice in enumerate(response_obj.get("choices")):
-                        if choice.get("finish_reason"):
-                            finish_reasons.append(choice.get("finish_reason"))
+                    choice_views: Final = [
+                        require_attributes(choice, _SupportsGet, "get")
+                        for choice in iter_items(response_view.get("choices"))
+                    ]
+                    finish_reasons: Final = [
+                        choice.get("finish_reason") for choice in choice_views if choice.get("finish_reason")
+                    ]
 
                     if finish_reasons:
                         self.safe_set_attribute(
@@ -2734,11 +2749,11 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
                             value=safe_dumps(finish_reasons),
                         )
 
-                    for idx, choice in enumerate(response_obj.get("choices")):
+                    for choice in choice_views:
                         if choice.get("finish_reason"):
-                            message = choice.get("message")
-                            if tool_calls := tool_calls_with_server_tool_calls(message):
-                                kv_pairs = OpenTelemetry._tool_calls_kv_pair(list(tool_calls))
+                            tool_calls = tool_calls_with_server_tool_calls(choice.get("message"))
+                            if tool_calls:
+                                kv_pairs = OpenTelemetry._tool_calls_kv_pair(tool_calls)
                                 for key, value in kv_pairs.items():
                                     self.safe_set_attribute(
                                         span=span,
@@ -2746,13 +2761,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
                                         value=value,
                                     )
 
-                elif response_obj.get("output"):
+                elif response_view.get("output"):
                     # Responses API: ResponsesAPIResponse has an "output"
                     # list instead of "choices".  Each item with
                     # type="message" contains a "content" list of
                     # OutputText objects (type="output_text").
-                    output_items: Final = response_obj.get("output")
-                    output_messages: Final = self._transform_responses_api_output_to_otel(output_items)
+                    output_items: Final = response_view.get("output")
+                    output_messages: Final = self._transform_responses_api_output_to_otel(iter_items(output_items))
                     if output_messages:
                         self.safe_set_attribute(
                             span=span,
@@ -2766,7 +2781,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
                     # ChatCompletionMessageToolCall format expected by
                     # _tool_calls_kv_pair.
                     tool_calls = []
-                    for out_item in output_items:
+                    for out_item in iter_items(output_items):
                         item_d = self._to_dict(out_item)
                         if item_d and item_d.get("type") == "function_call":
                             tool_calls.append(
@@ -2787,7 +2802,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
                             )
 
                     # Extract finish reason from ResponsesAPIResponse.status
-                    status: Final = response_obj.get("status")
+                    status: Final = response_view.get("status")
                     if status:
                         self.safe_set_attribute(
                             span=span,
@@ -2907,7 +2922,7 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             return obj.model_dump()
         return None
 
-    def _transform_responses_api_output_to_otel(self, output: list) -> list[dict]:
+    def _transform_responses_api_output_to_otel(self, output: Iterable[object]) -> list[dict]:
         """
         Transform Responses API output items into OTEL GenAI 1.38 format.
 

litellm/integrations/opentelemetry_utils/base_otel_llm_obs_attributes.py

@@ -1,4 +1,5 @@
 from abc import ABC
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 if TYPE_CHECKING:
@@ -7,7 +8,7 @@ if TYPE_CHECKING:
 
 class BaseLLMObsOTELAttributes(ABC):
     @staticmethod
-    def set_messages(span: "Span", kwargs: dict[str, Any]):
+    def set_messages(span: "Span", kwargs: Mapping[str, object]):
         pass
 
     @staticmethod

litellm/integrations/otel/logger.py

@@ -1034,10 +1034,12 @@ def emit_guardrail_span(entry: "StandardLoggingGuardrailInformation") -> None:
         pass
 
 
-def seed_request_identity(user_api_key_dict: object, model: str | None = None) -> None:
+def seed_request_identity(user_api_key_dict: object, model: str | None = None, request_data: object = None) -> None:
     logger: Final = _registered_v2_logger()
     if logger is not None:
-        logger.seed_request_identity(user_api_key_dict, model=model)
+        logger.seed_request_identity(
+            user_api_key_dict, model=model, request_metadata=metadata_from_request_data(request_data)
+        )
 
 
 def fan_out_provider() -> ApiTracerProvider:

litellm/integrations/otel/plumbing/metrics.py

@@ -11,6 +11,7 @@ identical metrics. The attribute cardinality filter is reused from v1 by import
 from collections.abc import Mapping
 from dataclasses import dataclass
 from datetime import datetime
+from types import MappingProxyType
 from typing import Any, Final, Literal, Protocol, TypeAlias
 
 from opentelemetry.metrics import Histogram, Meter
@@ -34,6 +35,7 @@ from litellm.integrations.otel.model.semconv import (
     resolve_provider,
 )
 from litellm.integrations.otel.model.utils import to_seconds
+from litellm.litellm_core_utils.core_helpers import attribute_or_none, require_attributes
 from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 
@@ -177,7 +179,14 @@ class _MetricKwargs(TypedDict, total=False):
     end_time: ReadOnly[datetime | float | str | None]
 
 
-def resolve_error_type(kwargs: Mapping[str, Any]) -> str:
+class _SupportsGet(Protocol):
+    def get(self, key: str, /) -> object: ...
+
+
+_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
+
+
+def resolve_error_type(kwargs: Mapping[str, object]) -> str:
     """The ``error.type`` value for a failed request.
 
     Bounded by construction: the mapped provider exception's class name (the same
@@ -188,8 +197,13 @@ def resolve_error_type(kwargs: Mapping[str, Any]) -> str:
     span and its exception event, where high cardinality is free.
     """
     std_log: Final = kwargs.get("standard_logging_object")
-    info: Final = getattr(std_log, "error_information", None) or (std_log or {}).get("error_information") or {}
-    error_class: Final = info.get("error_class") or info.get("error_code")
+    info: Final = (
+        attribute_or_none(std_log, "error_information")
+        or require_attributes(std_log or _NO_FIELDS, _SupportsGet, "get").get("error_information")
+        or _NO_FIELDS
+    )
+    info_fields: Final = require_attributes(info, _SupportsGet, "get")
+    error_class: Final = info_fields.get("error_class") or info_fields.get("error_code")
     if error_class:
         return str(error_class)
     exception: Final = kwargs.get("exception")

litellm/integrations/otel/runtime.py

@@ -67,12 +67,12 @@ def phase_event(name: str, attributes: PhaseEventAttributes | None = None) -> No
     runtime[2](name, attributes)
 
 
-def seed_request_identity(user_api_key_dict: object, model: object = None) -> None:
-    """Seed request-identity Baggage at the auth boundary (no-op without V2)."""
+def seed_request_identity(user_api_key_dict: object, model: object = None, request_data: object = None) -> None:
+    """Seed request-identity Baggage (no-op without V2)."""
     runtime: Final = _otel_runtime()
     if runtime is None:
         return
-    runtime[1](user_api_key_dict, model=model)
+    runtime[1](user_api_key_dict, model=model, request_data=request_data)
 
 
 def phase_attributes(attributes: PhaseAttributes) -> None:

litellm/integrations/prompt_layer.py

@@ -4,9 +4,12 @@ import os
 import traceback
 from typing import Final
 
-from pydantic import BaseModel
+from pydantic import BaseModel, TypeAdapter
 
 import litellm
+from litellm.litellm_core_utils.core_helpers import decoded_json_object
+
+_DECODED_JSON: Final = TypeAdapter(object)
 
 
 class PromptLayerLogger:
@@ -61,8 +64,8 @@ class PromptLayerLogger:
                 },
             )
 
-            response_json: Final = request_response.json()
-            if not request_response.json().get("success", False):
+            response_json: Final = decoded_json_object(_DECODED_JSON.validate_python(request_response.json()))
+            if not response_json.get("success", False):
                 raise Exception("Promptlayer did not successfully log the response!")
 
             print_verbose(f"Prompt Layer Logging: success - final response object: {request_response.text}")

litellm/integrations/weave/weave_otel.py

@@ -4,7 +4,8 @@ import base64
 import json
 import os
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, Final
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, Protocol
 
 from opentelemetry.trace import Status, StatusCode
 from typing_extensions import override
@@ -19,6 +20,7 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor
     BaseLLMObsOTELAttributes,
     safe_set_attribute,
 )
+from litellm.litellm_core_utils.core_helpers import has_attributes, require_attributes
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.types.integrations.weave_otel import WeaveOtelConfig, WeaveSpanAttributes
 from litellm.types.utils import StandardCallbackDynamicParams
@@ -34,6 +36,21 @@ WEAVE_BASE_URL: Final = "https://trace.wandb.ai"
 WEAVE_OTEL_ENDPOINT: Final = "/otel/v1/traces"
 
 
+_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
+
+
+class _SupportsGet(Protocol):
+    def get(self, key: str, /) -> object: ...
+
+
+class _SupportsReplace(Protocol):
+    def replace(self, old: str, new: str, /) -> object: ...
+
+
+class _ModelDumpable(Protocol):
+    def model_dump(self) -> object: ...
+
+
 class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
     """
     Weave-specific LLM observability OTEL attributes.
@@ -44,15 +61,15 @@ class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
 
     @staticmethod
     @override
-    def set_messages(span: Span, kwargs: dict[str, Any]):
+    def set_messages(span: Span, kwargs: Mapping[str, object]):
         """Set input messages as span attributes using OpenInference conventions."""
 
         messages: Final = kwargs.get("messages") or []
-        optional_params: Final = kwargs.get("optional_params") or {}
+        optional_params: Final = kwargs.get("optional_params") or _NO_FIELDS
 
         prompt: Final = {"messages": messages}
-        functions: Final = optional_params.get("functions")
-        tools: Final = optional_params.get("tools")
+        functions: Final = require_attributes(optional_params, _SupportsGet, "get").get("functions")
+        tools: Final = require_attributes(optional_params, _SupportsGet, "get").get("tools")
         if functions is not None:
             prompt["functions"] = functions
         if tools is not None:
@@ -60,7 +77,7 @@ class WeaveLLMObsOTELAttributes(BaseLLMObsOTELAttributes):
         safe_set_attribute(span, OpenInferenceSpanAttributes.INPUT_VALUE, json.dumps(prompt))
 
 
-def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, Any], response_obj: Any):
+def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, object], response_obj: object):
     """
     Sets Weave-specific metadata attributes onto the OTEL span.
 
@@ -69,25 +86,27 @@ def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, Any], respon
     """
 
     # Extract all needed data upfront
-    litellm_params: Final = kwargs.get("litellm_params") or {}
+    litellm_params: Final = kwargs.get("litellm_params") or _NO_FIELDS
     # optional_params = kwargs.get("optional_params") or {}
-    metadata: Final = kwargs.get("metadata") or {}
+    metadata: Final = kwargs.get("metadata") or _NO_FIELDS
     model: Final = kwargs.get("model") or ""
-    custom_llm_provider: Final = litellm_params.get("custom_llm_provider") or ""
+    custom_llm_provider: Final = (
+        require_attributes(litellm_params, _SupportsGet, "get").get("custom_llm_provider") or ""
+    )
 
     # Weave supports a custom display name and will default to the model name if not provided.
-    display_name = metadata.get("display_name")
+    display_name = require_attributes(metadata, _SupportsGet, "get").get("display_name")
     if not display_name and model:
         if custom_llm_provider:
             display_name = f"{custom_llm_provider}/{model}"
         else:
             display_name = model
     if display_name:
-        display_name = display_name.replace("/", "__")
+        display_name = require_attributes(display_name, _SupportsReplace, "replace").replace("/", "__")
         safe_set_attribute(span, WeaveSpanAttributes.DISPLAY_NAME.value, display_name)
 
     # Weave threads are OpenInference sessions.
-    if (session_id := metadata.get("session_id")) is not None:
+    if (session_id := require_attributes(metadata, _SupportsGet, "get").get("session_id")) is not None:
         if isinstance(session_id, (list, dict)):
             session_id = safe_dumps(session_id)
         safe_set_attribute(span, WeaveSpanAttributes.THREAD_ID.value, session_id)
@@ -97,7 +116,7 @@ def _set_weave_specific_attributes(span: Span, kwargs: Mapping[str, Any], respon
     # but we override them here to better match Weave's expectations
     if response_obj:
         output_dict = None
-        if hasattr(response_obj, "model_dump"):
+        if has_attributes(response_obj, _ModelDumpable, "model_dump"):
             output_dict = response_obj.model_dump()
         elif hasattr(response_obj, "get"):
             output_dict = response_obj

litellm/integrations/websearch_interception/tools.py

@@ -7,9 +7,19 @@ to this format for consistent interception and execution.
 """
 
 from collections.abc import Mapping
-from typing import Any, Final
+from typing import Final, Protocol
 
 from litellm.constants import LITELLM_WEB_SEARCH_TOOL_NAME
+from litellm.litellm_core_utils.core_helpers import require_attributes
+
+
+class _SupportsGetWithDefault(Protocol):
+    def get(self, key: str, default: str, /) -> object: ...
+
+
+class _SupportsStartswith(Protocol):
+    def startswith(self, prefix: str, /) -> object: ...
+
 
 _WEB_SEARCH_TOOL_DESCRIPTION: Final = (
     "Search the web for information. Use this when you need current "
@@ -164,7 +174,7 @@ def is_chat_completion_request_web_search_tool(tool: Mapping[str, object]) -> bo
     return tool.get("type") == "web_search" or is_web_search_tool(dict(tool))
 
 
-def is_web_search_tool_chat_completion(tool: dict[str, Any]) -> bool:
+def is_web_search_tool_chat_completion(tool: Mapping[str, object]) -> bool:
     """
     Check if a tool is a web search tool for Chat Completions API (strict check).
 
@@ -197,7 +207,7 @@ def is_web_search_tool_chat_completion(tool: dict[str, Any]) -> bool:
     # Check for OpenAI format: {"type": "function", "function": {"name": "litellm_web_search"}}
     if tool_type == "function" and "function" in tool:
         function_def: Final = tool.get("function", {})
-        function_name: Final = function_def.get("name", "")
+        function_name: Final = require_attributes(function_def, _SupportsGetWithDefault, "get").get("name", "")
         if function_name == LITELLM_WEB_SEARCH_TOOL_NAME:
             return True
 
@@ -229,7 +239,7 @@ def is_anthropic_native_web_search_tool(tool: Mapping[str, object]) -> bool:
     return tool_type.startswith("web_search_") and tool_type != "function"
 
 
-def is_web_search_tool(tool: dict[str, Any]) -> bool:
+def is_web_search_tool(tool: Mapping[str, object]) -> bool:
     """
     Check if a tool is a web search tool (native or LiteLLM standard).
 
@@ -281,7 +291,7 @@ def is_web_search_tool(tool: dict[str, Any]) -> bool:
     # Check for OpenAI format: {"type": "function", "function": {"name": "..."}}
     if tool_type == "function" and "function" in tool:
         function_def: Final = tool.get("function", {})
-        function_name: Final = function_def.get("name", "")
+        function_name: Final = require_attributes(function_def, _SupportsGetWithDefault, "get").get("name", "")
         if function_name == LITELLM_WEB_SEARCH_TOOL_NAME:
             return True
 
@@ -290,7 +300,7 @@ def is_web_search_tool(tool: dict[str, Any]) -> bool:
         return True
 
     # Check for native Anthropic web_search_* types
-    if tool_type.startswith("web_search_"):
+    if require_attributes(tool_type, _SupportsStartswith, "startswith").startswith("web_search_"):
         return True
 
     # Check for Claude Code's web_search with a type field

litellm/integrations/weights_biases.py

@@ -1,11 +1,13 @@
 from typing import Final
 
+from litellm.litellm_core_utils.core_helpers import iter_items, subscript
+
 imported_openAIResponse = True
 try:
     import io
     import logging
     from collections.abc import Mapping
-    from typing import Any, Literal, Protocol, TypeVar
+    from typing import Literal, Protocol, TypeVar
 
     from wandb.sdk.data_types import trace_tree
 
@@ -25,7 +27,7 @@ try:
     class OpenAIRequestResponseResolver:
         def __call__(
             self,
-            request: dict[str, Any],
+            request: Mapping[str, object],
             response: OpenAIResponse,
             time_elapsed: float,
         ) -> trace_tree.WBTraceTree | None:
@@ -110,14 +112,14 @@ try:
 
         def _resolve_chat_completion(
             self,
-            request: dict[str, Any],
+            request: Mapping[str, object],
             response: OpenAIResponse,
             time_elapsed: float,
         ) -> trace_tree.WBTraceTree:
             """Resolves the request and response objects for `openai.Completion`."""
             prompt: Final = io.StringIO()
-            for message in request["messages"]:
-                prompt.write(f"\n\n**{message['role']}**: {message['content']}\n")
+            for message in iter_items(request["messages"]):
+                prompt.write(f"\n\n**{subscript(message, 'role')}**: {subscript(message, 'content')}\n")
             request_str: Final = prompt.getvalue()
 
             choices: Final = [

litellm/interactions/litellm_responses_transformation/transformation.py

@@ -8,11 +8,11 @@ This module handles transforming between:
 
 from collections.abc import Mapping, Sequence
 from types import MappingProxyType
-from typing import Final, cast
+from typing import Final, Protocol, cast
 
 from pydantic import BaseModel
 
-from litellm.litellm_core_utils.core_helpers import billed_usage_cost
+from litellm.litellm_core_utils.core_helpers import attribute_or_none, billed_usage_cost, has_attributes
 from litellm.types.interactions import (
     InteractionInput,
     InteractionsAPIOptionalRequestParams,
@@ -27,6 +27,10 @@ from litellm.types.llms.openai import (
 _STEP_TYPE_ROLES: Final = MappingProxyType({"user_input": "user", "model_output": "assistant"})
 
 
+class _IsoFormattable(Protocol):
+    def isoformat(self) -> object: ...
+
+
 class LiteLLMResponsesInteractionsConfig:
     """Configuration class for transforming between Interactions API and Responses API."""
 
@@ -232,12 +236,12 @@ class LiteLLMResponsesInteractionsConfig:
                         )
 
         # Convert created_at to ISO string
-        created_at: Final = getattr(responses_response, "created_at", None)
+        created_at: Final = attribute_or_none(responses_response, "created_at")
         if isinstance(created_at, int):
             from datetime import datetime
 
             created = datetime.fromtimestamp(created_at).isoformat()
-        elif created_at is not None and hasattr(created_at, "isoformat"):
+        elif created_at is not None and has_attributes(created_at, _IsoFormattable, "isoformat"):
             created = created_at.isoformat()
         else:
             created = None

litellm/interactions/streaming_iterator.py

@@ -7,8 +7,9 @@ from the Google Interactions API, similar to the responses API streaming iterato
 
 import asyncio
 import json
+from collections.abc import Mapping
 from datetime import datetime
-from typing import Any, Final
+from typing import Final, Protocol
 
 import httpx
 from pydantic import ConfigDict, TypeAdapter
@@ -16,7 +17,7 @@ from pydantic import ConfigDict, TypeAdapter
 from litellm._logging import verbose_logger
 from litellm.constants import STREAM_SSE_DONE_STRING
 from litellm.litellm_core_utils.asyncify import run_async_function
-from litellm.litellm_core_utils.core_helpers import process_response_headers
+from litellm.litellm_core_utils.core_helpers import process_response_headers, require_attributes
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
 from litellm.litellm_core_utils.thread_pool_executor import executor
@@ -32,6 +33,10 @@ _INTERACTIONS_STREAMING_CHUNK_ADAPTER: Final = TypeAdapter(
 )
 
 
+class _SupportsGet(Protocol):
+    def get(self, key: str, default: object, /) -> object: ...
+
+
 class BaseInteractionsAPIStreamingIterator:
     """
     Base class for streaming iterators that process responses from the Interactions API.
@@ -45,7 +50,7 @@ class BaseInteractionsAPIStreamingIterator:
         model: str | None,
         interactions_api_config: BaseInteractionsAPIConfig,
         logging_obj: LiteLLMLoggingObj,
-        litellm_metadata: dict[str, Any] | None = None,
+        litellm_metadata: Mapping[str, object] | None = None,
         custom_llm_provider: str | None = None,
     ):
         self.response = response
@@ -65,9 +70,9 @@ class BaseInteractionsAPIStreamingIterator:
             model=model or "",
             optional_params=self.logging_obj.model_call_details.get("litellm_params", {}),
         )
-        _model_info: Final[dict] = litellm_metadata.get("model_info", {}) if litellm_metadata else {}
+        _model_info: Final[object] = litellm_metadata.get("model_info", {}) if litellm_metadata else {}
         self._hidden_params = {
-            "model_id": _model_info.get("id", None),
+            "model_id": require_attributes(_model_info, _SupportsGet, "get").get("id", None),
             "api_base": _api_base,
         }
         self._hidden_params["additional_headers"] = process_response_headers(self.response.headers or {})
@@ -135,7 +140,7 @@ class InteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator):
         model: str | None,
         interactions_api_config: BaseInteractionsAPIConfig,
         logging_obj: LiteLLMLoggingObj,
-        litellm_metadata: dict[str, Any] | None = None,
+        litellm_metadata: Mapping[str, object] | None = None,
         custom_llm_provider: str | None = None,
     ):
         super().__init__(
@@ -202,7 +207,7 @@ class SyncInteractionsAPIStreamingIterator(BaseInteractionsAPIStreamingIterator)
         model: str | None,
         interactions_api_config: BaseInteractionsAPIConfig,
         logging_obj: LiteLLMLoggingObj,
-        litellm_metadata: dict[str, Any] | None = None,
+        litellm_metadata: Mapping[str, object] | None = None,
         custom_llm_provider: str | None = None,
     ):
         super().__init__(

litellm/litellm_core_utils/audio_utils/utils.py

@@ -313,7 +313,7 @@ def calculate_request_duration(file: FileTypes) -> float | None:
                     file_content = content
                 elif hasattr(content, "read") and not isinstance(content, (str, os.PathLike)):
                     # File-like object in tuple
-                    current_pos: Final = getattr(content, "tell", lambda: None)()
+                    current_pos: Final = content.tell() if hasattr(content, "tell") else None
                     # Seek to start to ensure we read the entire content
                     if hasattr(content, "seek"):
                         content.seek(0)

litellm/litellm_core_utils/core_helpers.py

@@ -4,6 +4,7 @@ import copy
 import logging
 import math
 import re
+import sys
 from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
@@ -133,7 +134,7 @@ class _SupportsIter(Protocol):
 
 @runtime_checkable
 class _SupportsGetItem(Protocol):
-    def __getitem__(self, index: int, /) -> object: ...
+    def __getitem__(self, key: str | int, /) -> object: ...
 
 
 def attribute_or_none(source: object, name: str) -> object:
@@ -166,6 +167,28 @@ def iter_items(value: object) -> Iterable[object]:
     raise TypeError(f"'{type(value).__name__}' object is not iterable")
 
 
+@runtime_checkable
+class _SupportsContains(Protocol):
+    def __contains__(self, key: object, /) -> bool: ...
+
+
+_NOT_A_CONTAINER: Final = "is not a container or iterable" if sys.version_info >= (3, 14) else "is not iterable"
+
+
+def json_contains(value: object, key: object) -> bool:
+    if isinstance(value, _SupportsContains):
+        return key in value
+    if isinstance(value, (_SupportsIter, _SupportsGetItem)):
+        return key in tuple(iter_items(value))
+    raise TypeError(f"argument of type '{type(value).__name__}' {_NOT_A_CONTAINER}")
+
+
+def subscript(value: object, key: str | int) -> object:
+    if isinstance(value, _SupportsGetItem):
+        return value[key]
+    raise TypeError(f"'{type(value).__name__}' object is not subscriptable")
+
+
 def attribute_items(source: object, name: str) -> Iterable[object]:
     value: Final = attribute_or_none(source, name)
     return iter_items(value) if value else ()

litellm/litellm_core_utils/dd_tracing.py

@@ -5,8 +5,9 @@ If the ddtrace package is not installed, the tracer will be a no-op.
 """
 
 from contextlib import contextmanager
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any, Final, Protocol
 
+from litellm.litellm_core_utils.core_helpers import has_attributes
 from litellm.secret_managers.main import get_secret_bool
 
 if TYPE_CHECKING:
@@ -15,6 +16,14 @@ else:
     DD_TRACER = Any
 
 
+class _TagStrSpan(Protocol):
+    def set_tag_str(self, key: str, value: str, /) -> object: ...
+
+
+class _TagSpan(Protocol):
+    def set_tag(self, key: str, value: str, /) -> object: ...
+
+
 class NullSpan:
     """A no-op span implementation."""
 
@@ -81,7 +90,7 @@ else:
     tracer = NullTracer()
 
 
-def get_active_span() -> Any | None:
+def get_active_span() -> object:
     """
     Return the active Datadog span, checking current span first and then root span.
     """
@@ -115,10 +124,10 @@ def set_active_span_tag(tag_key: str, tag_value: str) -> bool:
         return False
 
     try:
-        if hasattr(span, "set_tag_str"):
+        if has_attributes(span, _TagStrSpan, "set_tag_str"):
             span.set_tag_str(tag_key, str(tag_value))
             return True
-        if hasattr(span, "set_tag"):
+        if has_attributes(span, _TagSpan, "set_tag"):
             span.set_tag(tag_key, str(tag_value))
             return True
     except Exception:

litellm/litellm_core_utils/litellm_logging.py

@@ -19,7 +19,7 @@ from datetime import datetime as dt_object
 from enum import Enum
 from functools import lru_cache
 from types import MappingProxyType, TracebackType
-from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, TypeVar, cast
+from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, Protocol, TypeVar, cast
 
 from httpx import Response, Timeout
 from openai import AsyncStream as OpenAIAsyncStream
@@ -90,6 +90,7 @@ from litellm.litellm_core_utils.classifier_logging import (
     is_classifier_call,
 )
 from litellm.litellm_core_utils.core_helpers import (
+    as_str_mapping,
     finite_cost,
     get_provider_response_headers_from_hidden_params,
     is_expected_client_error,
@@ -99,6 +100,7 @@ from litellm.litellm_core_utils.core_helpers import (
     proxy_stamped_used_client_oauth_token,
     reconstruct_model_name,
     reports_native_messages_cost,
+    require_attributes,
     set_response_cost_in_hidden_params,
     single_log_line,
 )
@@ -666,6 +668,31 @@ class UsageObservation(Enum):
     ESTIMATED = "estimated"
 
 
+class _A2AUsageSlot(Protocol):
+    usage: Mapping[str, object] | None
+
+
+class _A2AResponseCopier(Protocol):
+    model_copy: Callable[..., _A2AUsageSlot]
+
+
+class _MetadataLookup(Protocol):
+    get: Callable[[str], object]
+
+
+class _ParamsLookup(Protocol):
+    def get(self, key: str, default: object, /) -> object: ...
+
+
+def _callback_control_params(litellm_params: object) -> dict[str, object]:
+    mapping: Final = as_str_mapping(litellm_params)
+    if isinstance(mapping, dict):
+        return mapping
+    if mapping is not None:
+        return dict(mapping)
+    return litellm_params.model_dump() if isinstance(litellm_params, BaseModel) else {}
+
+
 class Logging(LiteLLMLoggingBaseClass):
     global \
         supabaseClient, \
@@ -929,7 +956,9 @@ class Logging(LiteLLMLoggingBaseClass):
         new_call.start_time = start_time
         new_call.litellm_params = litellm_params
         new_call.model_call_details = {
-            **{key: value for key, value in self.model_call_details.items() if not key.startswith("_accounting_")},
+            **{
+                key: value for key, value in self._model_call_details_view.items() if not key.startswith("_accounting_")
+            },
             "litellm_call_id": litellm_call_id,
             "start_time": start_time,
             "first_api_call_start_time": None,
@@ -2467,7 +2496,7 @@ class Logging(LiteLLMLoggingBaseClass):
         return self.response_cost_calculator(result=result, cache_hit=cache_hit)
 
     @staticmethod
-    def is_sync_litellm_request(litellm_params: Mapping[str, object]) -> bool:
+    def is_sync_litellm_request(litellm_params: _ParamsLookup) -> bool:
         """True for sync SDK entrypoints (``completion``), false for async (``acompletion``, etc.)."""
         return (
             litellm_params.get(CallTypes.acompletion.value, False) is not True
@@ -2918,11 +2947,12 @@ class Logging(LiteLLMLoggingBaseClass):
         """Deprecated alias of mark_logging_complete, kept for callers of the old name"""
         self.mark_logging_complete(event_type=event_type)
 
-    def should_run_callback(self, callback: litellm.CALLBACK_TYPES, litellm_params: dict, event_hook: str) -> bool:
+    def should_run_callback(self, callback: litellm.CALLBACK_TYPES, litellm_params: object, event_hook: str) -> bool:
         if litellm.global_disable_no_log_param:
             return True
 
-        if litellm_params.get("no-log", False) is True:
+        request_params: Final = require_attributes(litellm_params, _ParamsLookup, "get")
+        if request_params.get("no-log", False) is True:
             # proxy cost tracking cal backs should run
 
             if not (isinstance(callback, CustomLogger) and "_PROXY_" in callback.__class__.__name__):
@@ -2932,7 +2962,7 @@ class Logging(LiteLLMLoggingBaseClass):
         # Check for dynamically disabled callbacks via headers
         if EnterpriseCallbackControls is not None and EnterpriseCallbackControls.is_callback_disabled_dynamically(
             callback=callback,
-            litellm_params=litellm_params,
+            litellm_params=_callback_control_params(litellm_params),
             standard_callback_dynamic_params=self.standard_callback_dynamic_params,
         ):
             verbose_logger.debug(
@@ -3643,8 +3673,8 @@ class Logging(LiteLLMLoggingBaseClass):
             build_logging_payload=True,
             publish_logging_payload=not for_async_request,
         )
-        litellm_params = self.model_call_details.get("litellm_params", {})
-        is_sync_request: Final = self.is_sync_litellm_request(litellm_params)
+        litellm_params = self._model_call_details_view.get("litellm_params", {})
+        is_sync_request: Final = self.is_sync_litellm_request(require_attributes(litellm_params, _ParamsLookup, "get"))
         try:
             ## BUILD COMPLETE STREAMED RESPONSE
             complete_streaming_response: (
@@ -3800,18 +3830,23 @@ class Logging(LiteLLMLoggingBaseClass):
 
                         model = call_details_view["model"]
                         messages = call_details_view["messages"]
-                        optional_params = kwargs.get("optional_params", {})
-                        litellm_params = kwargs.get("litellm_params", {})
+                        optional_params = call_details_view.get("optional_params", {})
+                        litellm_params = call_details_view.get("litellm_params", {})
                         supabaseClient.log_event(
                             model=model,
                             messages=messages,
-                            end_user=optional_params.get("user", "default"),
+                            end_user=require_attributes(optional_params, _ParamsLookup, "get").get("user", "default"),
                             response_obj=result,
                             start_time=start_time,
                             end_time=end_time,
                             litellm_call_id=(
                                 current_call_id
-                                if (current_call_id := litellm_params.get("litellm_call_id")) is not None
+                                if (
+                                    current_call_id := require_attributes(litellm_params, _ParamsLookup, "get").get(
+                                        "litellm_call_id", None
+                                    )
+                                )
+                                is not None
                                 else str(uuid.uuid4())
                             ),
                             print_verbose=print_verbose,
@@ -4369,7 +4404,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
         for callback in delivery_callbacks:
             # check if callback can run for this request
-            litellm_params = self.model_call_details.get("litellm_params", {})
+            litellm_params = self._model_call_details_view.get("litellm_params", {})
             should_run = self.should_run_callback(
                 callback=callback,
                 litellm_params=litellm_params,
@@ -4658,8 +4693,8 @@ class Logging(LiteLLMLoggingBaseClass):
         verbose_logger.debug("Logging Details LiteLLM-Failure Call: %s", litellm.failure_callback)
         if not self.should_run_logging(event_type="sync_failure"):  # prevent double logging
             return
-        litellm_params: Final = self.model_call_details.get("litellm_params", {})
-        is_sync_request: Final = self.is_sync_litellm_request(litellm_params)
+        litellm_params: Final = self._model_call_details_view.get("litellm_params", {})
+        is_sync_request: Final = self.is_sync_litellm_request(require_attributes(litellm_params, _ParamsLookup, "get"))
 
         try:
             start_time, end_time = self._failure_handler_helper_fn(
@@ -4879,7 +4914,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self.mark_logging_complete(event_type="async_failure")
         for callback in callbacks:
             try:
-                litellm_params = self.model_call_details.get("litellm_params", {})
+                litellm_params = self._model_call_details_view.get("litellm_params", {})
                 should_run = self.should_run_callback(
                     callback=callback,
                     litellm_params=litellm_params,
@@ -5441,7 +5476,7 @@ class Logging(LiteLLMLoggingBaseClass):
         httpx_response: Final[Response | None] = self.model_call_details.get("httpx_response", None)
         if httpx_response is None:
             raise ValueError("Google GenAI Generate Content: httpx_response is None")
-        dict_result: Final = httpx_response.json()
+        dict_result: Final[GenerateContentResponseBody] = httpx_response.json()
         result = litellm.VertexGeminiConfig().transform_google_generate_content_to_openai_model_response(
             completion_response=dict_result,
             model_response=litellm.ModelResponse(),
@@ -5454,7 +5489,7 @@ class Logging(LiteLLMLoggingBaseClass):
         )
         return result
 
-    def _handle_a2a_response_logging(self, result: Any) -> object:
+    def _handle_a2a_response_logging(self, result: object) -> object:
         """
         Handles logging for A2A (Agent-to-Agent) responses.
 
@@ -5473,7 +5508,7 @@ class Logging(LiteLLMLoggingBaseClass):
             return result
 
         # Deep copy result and add usage
-        result_copy: Final = result.model_copy(deep=True)
+        result_copy: Final = require_attributes(result, _A2AResponseCopier, "model_copy").model_copy(deep=True)
         result_copy.usage = usage.model_dump() if hasattr(usage, "model_dump") else dict(usage)
         return result_copy
 
@@ -7475,7 +7510,7 @@ class StandardLoggingPayloadSetup:
     @staticmethod
     def get_standard_logging_payload_trace_id(
         logging_obj: Logging,
-        litellm_params: Mapping[str, Any],
+        litellm_params: Mapping[str, object],
     ) -> str:
         """
         Returns the `litellm_trace_id` for this request
@@ -7491,9 +7526,13 @@ class StandardLoggingPayloadSetup:
         """
         dynamic_litellm_session_id: Final = litellm_params.get("litellm_session_id")
         dynamic_litellm_trace_id: Final = litellm_params.get("litellm_trace_id")
-        metadata: Final[Mapping[str, object] | None] = litellm_params.get("metadata")
-        metadata_session_id: Final = metadata.get("session_id") if metadata else None
-        metadata_trace_id: Final = metadata.get("trace_id") if metadata else None
+        metadata: Final = litellm_params.get("metadata")
+        metadata_session_id: Final = (
+            require_attributes(metadata, _MetadataLookup, "get").get("session_id") if metadata else None
+        )
+        metadata_trace_id: Final = (
+            require_attributes(metadata, _MetadataLookup, "get").get("trace_id") if metadata else None
+        )
 
         ordered_candidates: Final[tuple[object, object, object, object]] = (
             (dynamic_litellm_trace_id, dynamic_litellm_session_id, metadata_trace_id, metadata_session_id)
@@ -7508,7 +7547,7 @@ class StandardLoggingPayloadSetup:
     @staticmethod
     def get_standard_logging_payload_session_id(
         logging_obj: Logging,
-        litellm_params: Mapping[str, Any],
+        litellm_params: Mapping[str, object],
     ) -> str:
         """
         Returns the end-user/conversation `litellm_session_id` for this request, independent of trace_id.
@@ -7523,8 +7562,10 @@ class StandardLoggingPayloadSetup:
         dynamic_litellm_session_id: Final[object] = litellm_params.get("litellm_session_id")
         if dynamic_litellm_session_id:
             return str(dynamic_litellm_session_id)
-        metadata: Final[Mapping[str, object] | None] = litellm_params.get("metadata")
-        metadata_session_id: Final = metadata.get("session_id") if metadata else None
+        metadata: Final = litellm_params.get("metadata")
+        metadata_session_id: Final = (
+            require_attributes(metadata, _MetadataLookup, "get").get("session_id") if metadata else None
+        )
         if metadata_session_id:
             return str(metadata_session_id)
         return logging_obj.litellm_session_id

litellm/litellm_core_utils/llm_cost_calc/usage_object_transformation.py

@@ -1,7 +1,8 @@
 from collections.abc import Mapping, Sequence
 from types import MappingProxyType
-from typing import Any, Final
+from typing import Any, Final, Protocol
 
+from litellm.litellm_core_utils.core_helpers import iter_items, require_attributes
 from litellm.types.utils import (
     CompletionTokensDetailsWrapper,
     PromptTokensDetailsWrapper,
@@ -55,19 +56,29 @@ _INTERACTIONS_MODALITY_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
 )
 
 
-def _modality_field(entry: Mapping[str, object]) -> str | None:
-    return _INTERACTIONS_MODALITY_FIELDS.get(str(entry.get("modality", "")).lower())
+class _SupportsGet(Protocol):
+    def get(self, key: str, default: object = None, /) -> object: ...
+
+
+def _modality_field(entry: object) -> str | None:
+    return _INTERACTIONS_MODALITY_FIELDS.get(
+        str(require_attributes(entry, _SupportsGet, "get").get("modality", "")).lower()
+    )
 
 
 def _token_count(value: object) -> int:
     return value if isinstance(value, int) else 0
 
 
-def _modality_token_sums(entries: Sequence[Mapping[str, object]]) -> Mapping[str, int]:
+def _modality_token_sums(entries: Sequence[object]) -> Mapping[str, int]:
     fields: Final = frozenset(field for entry in entries if (field := _modality_field(entry)) is not None)
     return MappingProxyType(
         {
-            field: sum(_token_count(entry.get("tokens")) for entry in entries if _modality_field(entry) == field)
+            field: sum(
+                _token_count(require_attributes(entry, _SupportsGet, "get").get("tokens"))
+                for entry in entries
+                if _modality_field(entry) == field
+            )
             for field in fields
         }
     )
@@ -119,12 +130,16 @@ class InteractionsUsageObjectTransformation:
         return "total_input_tokens" in usage_object or "total_output_tokens" in usage_object
 
     @staticmethod
-    def transform_interactions_usage_object(usage_object: Mapping[str, Any]) -> Usage:
-        input_entries: Final = tuple(usage_object.get("input_tokens_by_modality") or ()) + tuple(
-            usage_object.get("tool_use_tokens_by_modality") or ()
+    def transform_interactions_usage_object(usage_object: Mapping[str, object]) -> Usage:
+        input_entries: Final = tuple(iter_items(usage_object.get("input_tokens_by_modality") or ())) + tuple(
+            iter_items(usage_object.get("tool_use_tokens_by_modality") or ())
+        )
+        cached_sums: Final = _modality_token_sums(
+            tuple(iter_items(usage_object.get("cached_tokens_by_modality") or ()))
+        )
+        output_sums: Final = _modality_token_sums(
+            tuple(iter_items(usage_object.get("output_tokens_by_modality") or ()))
         )
-        cached_sums: Final = _modality_token_sums(tuple(usage_object.get("cached_tokens_by_modality") or ()))
-        output_sums: Final = _modality_token_sums(tuple(usage_object.get("output_tokens_by_modality") or ()))
 
         total_cached_tokens: Final = _token_count(usage_object.get("total_cached_tokens"))
         input_sums: Final = _subtract_cached_from_input(

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -6,10 +6,11 @@ from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field, replace
 from datetime import datetime, timezone, tzinfo
 from types import MappingProxyType
-from typing import Final, Literal, Protocol, TypedDict, cast
+from typing import Final, Literal, Protocol, SupportsFloat, SupportsIndex, TypedDict, cast
 from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
 
-from typing_extensions import ReadOnly
+from pydantic import TypeAdapter
+from typing_extensions import Buffer, ReadOnly
 
 import litellm
 from litellm._internal_context import current_billing_time
@@ -48,10 +49,10 @@ _IMAGE_RESPONSE_CALL_TYPES: Final = frozenset(
     }
 )
 
-# Pre-resolved DataResidency enum values for fast membership checks
 _VALID_DATA_RESIDENCIES: Final = frozenset(r.value for r in DataResidency)
 
 _DEPLOYMENT_PRICING_KEYS: Final[frozenset[str]] = frozenset(CustomPricingLiteLLMParams.model_fields)
+_IMAGE_REQUEST_PARAMS: Final[TypeAdapter[Mapping[str, object] | None]] = TypeAdapter(Mapping[str, object] | None)
 
 _BASE_CACHE_WRITE_RATE_KEY: Final = "cache_creation_input_token_cost"
 
@@ -1428,6 +1429,13 @@ def get_regional_uplift_multiplier(model_info: ModelInfo, data_residency: str |
     multiplier: Final = model_info.get(f"regional_processing_uplift_multiplier_{residency}")
     if multiplier is None:
         return 1.0
+    if not isinstance(multiplier, (str, Buffer, SupportsFloat, SupportsIndex)):
+        verbose_logger.error(
+            "Invalid regional_processing_uplift_multiplier_%s for model (got %s); defaulting to 1.0",
+            residency,
+            type(multiplier).__name__,
+        )
+        return 1.0
     try:
         return float(multiplier)
     except (TypeError, ValueError):
@@ -1464,7 +1472,7 @@ def get_fedramp_uplift_multiplier(model_info: ModelInfo, data_residency: str | N
         return 1.0
 
 
-def get_vertex_regional_endpoint_uplift(model_info: ModelInfo, vertex_location: str | None) -> float:
+def get_vertex_regional_endpoint_uplift(model_info: Mapping[str, object], vertex_location: str | None) -> float:
     """
     Resolve the per-model uplift multiplier for Vertex AI non-global (regional and
     multi-region) endpoints.
@@ -2318,6 +2326,20 @@ class CostCalculatorUtils:
         )
         resolved_n: Final = n if n is not None else (len(completion_response.data) if completion_response.data else 0)
         pricing: Final = deployment_pricing(model_info)
+        if custom_llm_provider == litellm.LlmProviders.XAI.value:
+            from litellm.llms.xai.image_generation.cost_calculator import cost_calculator as xai_image_cost_calculator
+
+            xai_cost: Final = xai_image_cost_calculator(
+                model=model,
+                image_response=completion_response,
+                quality=quality,
+                n=resolved_n,
+                optional_params=_IMAGE_REQUEST_PARAMS.validate_python(optional_params),
+                call_type=call_type,
+                model_info=pricing,
+            )
+            if xai_cost is not None:
+                return xai_cost
         model_priced_calculator: Final = _model_priced_image_cost_calculator(custom_llm_provider, model, pricing)
 
         if model_priced_calculator is not None:

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -6,7 +6,7 @@ import json
 from collections.abc import Hashable, Iterable, Mapping, Sequence
 from dataclasses import dataclass
 from itertools import chain, groupby
-from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
+from typing import TYPE_CHECKING, Final, Literal, TypeAlias, cast
 
 from pydantic import TypeAdapter
 
@@ -1143,7 +1143,7 @@ def strip_prefill_prefix_field(message: AllMessageValues) -> AllMessageValues:
     return _message_without_fields(message, frozenset({"prefix"}))
 
 
-def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
+def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Mapping[object, object]:
     """
     Filters a value from a dictionary
 

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -10,7 +10,7 @@ from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Se
 from dataclasses import dataclass, replace
 from enum import Enum
 from types import MappingProxyType
-from typing import Annotated, Any, Final, TypeAlias, TypedDict, TypeVar, cast, overload
+from typing import Annotated, Any, Final, Protocol, TypeAlias, TypedDict, TypeVar, cast, overload
 
 from jinja2.sandbox import ImmutableSandboxedEnvironment
 from pydantic import BaseModel, BeforeValidator, ConfigDict, JsonValue, TypeAdapter, ValidationError, model_validator
@@ -22,7 +22,7 @@ import litellm.types.llms
 from litellm import verbose_logger
 from litellm._uuid import uuid
 from litellm.constants import REDACTED_BY_LITELLM
-from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, require_attributes
 from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
     anthropic_system_messages,
     find_anthropic_server_tool_result,
@@ -4150,7 +4150,7 @@ class BedrockImageProcessor:
     _validate_format = validate_format
 
     @staticmethod
-    def _get_document_format(mime_type: str, supported_doc_formats: list[str]) -> str:
+    def _get_document_format(mime_type: str, supported_doc_formats: Sequence[str]) -> str:
         """
         Get the document format from the mime type
 
@@ -6571,6 +6571,20 @@ def has_tool_with_name(tools: object, tool_name: str) -> bool:
     return False
 
 
+class _SupportsModelDump(Protocol):
+    def model_dump(self) -> dict[str, object]: ...
+
+
+class _StructuredMessagesSource(Protocol):
+    def get_structured_messages(self, data: dict[str, object], /) -> Sequence[object] | None: ...
+
+
+def _instantiate_structured_message_sources(
+    source_classes: Iterable[Callable[[], _StructuredMessagesSource]],
+) -> Sequence[_StructuredMessagesSource]:
+    return [source_cls() for source_cls in source_classes]
+
+
 def resolve_structured_messages(
     messages: list[dict[str, object]] | None,
     request_kwargs: dict[str, Any],
@@ -6608,16 +6622,17 @@ def resolve_structured_messages(
                     break
 
     # 2. Fallback: try each mapped handler until one produces messages
-    handlers_to_try: Final[list[Any]] = []
-    if call_type is not None and call_type in mappings:
-        handlers_to_try.append(mappings[call_type]())
-    else:
-        handlers_to_try.extend(handler_cls() for handler_cls in mappings.values())
+    handlers_to_try: Final = _instantiate_structured_message_sources(
+        (mappings[call_type],) if call_type is not None and call_type in mappings else mappings.values()
+    )
 
     for handler in handlers_to_try:
         structured = handler.get_structured_messages(request_kwargs)
         if structured:
-            return [msg if isinstance(msg, dict) else msg.model_dump() for msg in structured]
+            return [
+                msg if isinstance(msg, dict) else require_attributes(msg, _SupportsModelDump, "model_dump").model_dump()
+                for msg in structured
+            ]
     return None
 
 

litellm/litellm_core_utils/prompt_templates/tpm_media_utils.py

@@ -202,7 +202,7 @@ def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection:
     part: Final = as_str_mapping(value)
     if part is not None and part.get("type") == "file":
         return _tpm_embedding_file_block(part)
-    if not isinstance(value, str) or not value.startswith("data:"):
+    if not isinstance(value, str) or is_plain_embedding_text(value):
         return _tpm_media_leaf(value)
     if not _is_base64_data_uri(value):
         return _TPMMediaProjection(value)
@@ -210,6 +210,10 @@ def _tpm_embedding_leaf(value: object) -> _TPMMediaProjection:
     return _TPMMediaProjection("" if media else value, media)
 
 
+def is_plain_embedding_text(value: object) -> bool:
+    return isinstance(value, str) and not value.startswith("data:")
+
+
 def _is_json_array(value: object) -> bool:
     return isinstance(value, (list, tuple))
 

litellm/litellm_core_utils/realtime_guardrails.py

@@ -0,0 +1,316 @@
+import json
+from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Final, TypeAlias
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm._logging import verbose_logger
+from litellm.exceptions import ModifyResponseException
+
+if TYPE_CHECKING:
+    from litellm.integrations.custom_guardrail import CustomGuardrail
+    from litellm.types.guardrails import GuardrailEventHooks
+
+_DEFAULT_VIOLATION_MESSAGE: Final = "I'm sorry, that request was blocked by the content filter."
+BLOCKED_TOOL_OUTPUT: Final = json.dumps({"error": "Tool output blocked by content policy"})
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
+_JSON_ARRAY: Final = TypeAdapter(tuple[object, ...])
+_TEXTS: Final = TypeAdapter(tuple[str, ...])
+_TYPED_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"})
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeGuardrailBlock:
+    message: str
+    ends_session_on_violation: bool
+    end_session_after_n_fails: int | None
+
+    def ends_session(self, violation_count: int) -> bool:
+        return self.ends_session_on_violation or (
+            self.end_session_after_n_fails is not None and violation_count >= self.end_session_after_n_fails
+        )
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeGuardrailRewrite:
+    texts: tuple[str, ...]
+
+
+RealtimeGuardrailVerdict: TypeAlias = RealtimeGuardrailBlock | RealtimeGuardrailRewrite | None
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeMessageText:
+    role: str
+    texts: tuple[str, ...]
+
+
+@dataclass(frozen=True, slots=True)
+class RealtimeToolOutput:
+    texts: tuple[str, ...]
+    sanitized_event: str
+
+
+RealtimeGuardedItem: TypeAlias = RealtimeMessageText | RealtimeToolOutput
+
+
+def _json_object(value: object) -> Mapping[str, object] | None:
+    try:
+        return _JSON_OBJECT.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def _turn_copy(value: object) -> object:
+    mapping: Final = _json_object(value)
+    return value if mapping is None else dict(mapping)
+
+
+def _typed_text(part: object) -> str | None:
+    part_object: Final = _json_object(part)
+    if part_object is None or part_object.get("type") not in _TYPED_TEXT_PART_TYPES:
+        return None
+    text: Final[object] = part_object.get("text", "")
+    return text if isinstance(text, str) else None
+
+
+def _content_parts(content: object) -> Sequence[object]:
+    return _JSON_ARRAY.validate_python(content) if isinstance(content, list) else ()
+
+
+def _typed_texts(content: object) -> tuple[str, ...]:
+    return tuple(text for text in map(_typed_text, _content_parts(content)) if text is not None)
+
+
+def _with_text(part: object, text: str | None) -> object:
+    part_object: Final = _json_object(part)
+    return part if text is None or part_object is None else {**part_object, "text": text}
+
+
+def _tool_output(client_event: Mapping[str, object], item: Mapping[str, object]) -> RealtimeToolOutput | None:
+    output: Final[object] = item.get("output", "")
+    text: Final = output if isinstance(output, str) else json.dumps(output)
+    if not text:
+        return None
+    sanitized_event: Final = json.dumps({**client_event, "item": {**item, "output": BLOCKED_TOOL_OUTPUT}})
+    return RealtimeToolOutput(texts=(text,), sanitized_event=sanitized_event)
+
+
+def realtime_guarded_item(client_event: Mapping[str, object]) -> RealtimeGuardedItem | None:
+    if client_event.get("type") != "conversation.item.create":
+        return None
+    item: Final = _json_object(client_event.get("item"))
+    if item is None:
+        return None
+    if item.get("type") == "function_call_output":
+        return _tool_output(client_event, item)
+    texts: Final = _typed_texts(item.get("content"))
+    role: Final[object] = item.get("role")
+    return RealtimeMessageText(role=role if isinstance(role, str) else "user", texts=texts) if any(texts) else None
+
+
+def rewritten_item(client_event: Mapping[str, object], texts: tuple[str, ...]) -> Mapping[str, object]:
+    item: Final = _json_object(client_event.get("item")) or {}
+    if item.get("type") == "function_call_output":
+        return {**item, "output": " ".join(texts)}
+    parts: Final = _content_parts(item.get("content"))
+    text_positions: Final = tuple(index for index, part in enumerate(parts) if _typed_text(part) is not None)
+    replacements: Final = dict(zip(text_positions, texts))
+    merged_away: Final = frozenset(text_positions[len(texts) :])
+    return {
+        **item,
+        "content": [
+            _with_text(part, replacements.get(index)) for index, part in enumerate(parts) if index not in merged_away
+        ],
+    }
+
+
+def _violation_text(error: Exception) -> str:
+    detail: Final[object] = getattr(error, "detail", None)
+    detail_object: Final = _json_object(detail)
+    if detail_object is not None:
+        reason: Final[object] = detail_object.get("error")
+        return reason if isinstance(reason, str) and reason else str(error)
+    if detail is not None:
+        return str(detail)
+    return str(error) or _DEFAULT_VIOLATION_MESSAGE
+
+
+def _is_guardrail_block(error: Exception) -> bool:
+    return hasattr(error, "status_code") or isinstance(error, (ValueError, ModifyResponseException))
+
+
+def _screened_texts(screened: object, texts: tuple[str, ...]) -> tuple[str, ...]:
+    screened_object: Final = _json_object(screened)
+    try:
+        returned: Final = _TEXTS.validate_python(screened_object.get("texts") if screened_object else None)
+    except ValidationError:
+        return texts
+    return returned if len(returned) == len(texts) else texts
+
+
+async def _guardrail_outcome(
+    guardrail: "CustomGuardrail", texts: tuple[str, ...], request_data: Mapping[str, object]
+) -> RealtimeGuardrailBlock | tuple[str, ...]:
+    try:
+        screened: Final = await guardrail.apply_guardrail(
+            inputs={"texts": list(texts), "images": []},
+            request_data=dict(request_data),
+            input_type="request",
+        )
+    except Exception as error:
+        if not _is_guardrail_block(error):
+            verbose_logger.exception("[realtime guardrail] unexpected error in apply_guardrail: %s", error)
+            raise
+        return RealtimeGuardrailBlock(
+            message=guardrail.realtime_violation_message or _violation_text(error),
+            ends_session_on_violation=guardrail.on_violation == "end_session",
+            end_session_after_n_fails=guardrail.end_session_after_n_fails,
+        )
+    return _screened_texts(screened, texts)
+
+
+async def _screen(
+    guardrails: Sequence["CustomGuardrail"], texts: tuple[str, ...], request_data: Mapping[str, object]
+) -> RealtimeGuardrailBlock | tuple[str, ...]:
+    if not guardrails:
+        return texts
+    outcome: Final = await _guardrail_outcome(guardrails[0], texts, request_data)
+    if isinstance(outcome, RealtimeGuardrailBlock):
+        return outcome
+    return await _screen(guardrails[1:], outcome, request_data)
+
+
+async def _screened_until_block(
+    guardrails: Sequence["CustomGuardrail"], texts: tuple[str, ...], request_data: Mapping[str, object]
+) -> AsyncIterator[RealtimeGuardrailBlock | str]:
+    for text in texts:
+        if isinstance(outcome := await _screen(guardrails, (text,), request_data), RealtimeGuardrailBlock):
+            yield outcome
+            return
+        for screened in outcome:
+            yield screened
+
+
+async def _screen_each(
+    guardrails: Sequence["CustomGuardrail"], texts: tuple[str, ...], request_data: Mapping[str, object]
+) -> RealtimeGuardrailBlock | tuple[str, ...]:
+    outcomes: Final = tuple([outcome async for outcome in _screened_until_block(guardrails, texts, request_data)])
+    block: Final = next((outcome for outcome in outcomes if isinstance(outcome, RealtimeGuardrailBlock)), None)
+    return block if block is not None else tuple(outcome for outcome in outcomes if isinstance(outcome, str))
+
+
+def _joins(texts: tuple[str, ...]) -> tuple[str, ...]:
+    unspaced: Final = "".join(texts)
+    boundaries: Final = zip(texts, texts[1:])
+    word_break_unknown: Final = any(not left[-1:].isspace() and not right[:1].isspace() for left, right in boundaries)
+    return (unspaced, " ".join(texts)) if word_break_unknown else (unspaced,)
+
+
+async def _rewritten_parts(
+    guardrails: Sequence["CustomGuardrail"], texts: tuple[str, ...], request_data: Mapping[str, object]
+) -> RealtimeGuardrailBlock | tuple[str, ...]:
+    parts: Final = await _screen_each(guardrails, texts, request_data)
+    if isinstance(parts, RealtimeGuardrailBlock):
+        return parts
+    rejoined: Final = _joins(parts)
+    screened_rejoined: Final = await _screen_each(guardrails, rejoined, request_data)
+    if isinstance(screened_rejoined, RealtimeGuardrailBlock):
+        return screened_rejoined
+    masked_join: Final = next((text for text, join in zip(screened_rejoined, rejoined) if text != join), None)
+    return parts if masked_join is None else (masked_join,)
+
+
+def _in_role_scope(guardrail: "CustomGuardrail", role: str) -> bool:
+    from litellm.llms.base_llm.guardrail_translation.utils import (
+        effective_scan_only_tool_results_for_guardrail,
+        effective_skip_system_message_for_guardrail,
+        effective_skip_tool_message_for_guardrail,
+        role_out_of_guardrail_scope,
+    )
+
+    return not role_out_of_guardrail_scope(
+        role,
+        skip_system_message=effective_skip_system_message_for_guardrail(guardrail),
+        skip_tool_message=effective_skip_tool_message_for_guardrail(guardrail),
+        scan_only_tool_results=effective_scan_only_tool_results_for_guardrail(guardrail),
+    )
+
+
+def realtime_runner_guardrail(
+    callback: object,
+    check_data: Mapping[str, object],
+    event_hooks: Sequence["GuardrailEventHooks"],
+) -> "CustomGuardrail | None":
+    from litellm.integrations.custom_guardrail import CustomGuardrail
+
+    if not isinstance(callback, CustomGuardrail) or callback.use_native_lifecycle_hooks:
+        return None
+    runs: Final = any(
+        callback.should_run_guardrail(data=check_data, event_type=event_hook) for event_hook in event_hooks
+    )
+    return callback if runs else None
+
+
+def realtime_screening_guardrails(
+    role: str,
+    event_hooks: Sequence["GuardrailEventHooks"],
+    check_data: Mapping[str, object],
+    callbacks: Iterable[object],
+) -> tuple["CustomGuardrail", ...]:
+    runner_guardrails: Final = (realtime_runner_guardrail(callback, check_data, event_hooks) for callback in callbacks)
+    guardrails: Final = {
+        id(guardrail): guardrail
+        for guardrail in runner_guardrails
+        if guardrail is not None and _in_role_scope(guardrail, role)
+    }
+    return tuple(guardrails.values())
+
+
+async def realtime_guardrail_verdict(
+    texts: tuple[str, ...],
+    role: str,
+    event_hooks: Sequence["GuardrailEventHooks"],
+    request_data: Mapping[str, object],
+    user_api_key_dict: object | None,
+    callbacks: Iterable[object],
+) -> RealtimeGuardrailVerdict:
+    check_data: Final = {**request_data, "transcript": " ".join(texts), "stream": True}
+    in_scope: Final = realtime_screening_guardrails(role, event_hooks, check_data, callbacks)
+    guardrail_request_data: Final = {
+        **{key: _turn_copy(value) for key, value in request_data.items()},
+        "user_api_key_dict": user_api_key_dict,
+    }
+    joined: Final = _joins(texts)
+    screened: Final = await _screen_each(in_scope, joined, guardrail_request_data)
+    if isinstance(screened, RealtimeGuardrailBlock):
+        return screened
+    if screened == joined:
+        return None
+    rewritten: Final = screened if len(texts) < 2 else await _rewritten_parts(in_scope, texts, guardrail_request_data)
+    if isinstance(rewritten, RealtimeGuardrailBlock):
+        return rewritten
+    return None if rewritten == texts else RealtimeGuardrailRewrite(texts=rewritten)
+
+
+def guardrail_violation_error_event(message: str) -> str:
+    return json.dumps(
+        {
+            "type": "error",
+            "error": {"type": "guardrail_violation", "message": message, "code": "content_policy_violation"},
+        }
+    )
+
+
+def guardrail_voice_prompt_events(message: str) -> tuple[str, str]:
+    prompt: Final = f"Say exactly the following message to the user, word for word, do not add anything else: {message}"
+    return (
+        json.dumps(
+            {
+                "type": "conversation.item.create",
+                "item": {"type": "message", "role": "user", "content": [{"type": "input_text", "text": prompt}]},
+            }
+        ),
+        json.dumps({"type": "response.create"}),
+    )

litellm/litellm_core_utils/realtime_streaming.py

@@ -47,13 +47,27 @@ from litellm.types.realtime import ALL_DELTA_TYPES
 
 from .litellm_logging import Logging as LiteLLMLogging
 from .realtime_errors import client_close_code, realtime_error_event, websocket_close_reason
+from .realtime_guardrails import (
+    RealtimeGuardedItem,
+    RealtimeGuardrailBlock,
+    RealtimeGuardrailRewrite,
+    RealtimeGuardrailVerdict,
+    RealtimeMessageText,
+    RealtimeToolOutput,
+    guardrail_violation_error_event,
+    guardrail_voice_prompt_events,
+    realtime_guarded_item,
+    realtime_guardrail_verdict,
+    realtime_runner_guardrail,
+    realtime_screening_guardrails,
+    rewritten_item,
+)
 
 if TYPE_CHECKING:
     from openai.types.realtime.realtime_audio_formats import RealtimeAudioFormats
     from websockets.asyncio.client import ClientConnection
     from websockets.exceptions import ConnectionClosed
 
-    from litellm.integrations.custom_guardrail import CustomGuardrail
     from litellm.types.guardrails import GuardrailEventHooks
 
     CLIENT_CONNECTION_CLASS = ClientConnection
@@ -437,21 +451,6 @@ def _opted_out_of_always_on_guardrail(callback: object, request_data: Mapping[st
     )
 
 
-def _realtime_runner_guardrail(
-    callback: object,
-    check_data: Mapping[str, object],
-    event_hooks: Sequence["GuardrailEventHooks"],
-) -> "CustomGuardrail | None":
-    from litellm.integrations.custom_guardrail import CustomGuardrail
-
-    if not isinstance(callback, CustomGuardrail) or callback.use_native_lifecycle_hooks:
-        return None
-    runs: Final = any(
-        callback.should_run_guardrail(data=check_data, event_type=event_hook) for event_hook in event_hooks
-    )
-    return callback if runs else None
-
-
 def translation_guardrails_apply(request_data: Mapping[str, object]) -> bool:
     from litellm.types.guardrails import GuardrailEventHooks
 
@@ -479,7 +478,7 @@ def realtime_guardrails_apply(request_data: Mapping[str, object], callbacks: Seq
     check_data: Final = {**request_data, "stream": True}
     runner_event_hooks: Final = (GuardrailEventHooks.realtime_input_transcription, GuardrailEventHooks.pre_call)
     return any(
-        _realtime_runner_guardrail(callback, check_data, runner_event_hooks) is not None for callback in callbacks
+        realtime_runner_guardrail(callback, check_data, runner_event_hooks) is not None for callback in callbacks
     )
 
 
@@ -1663,16 +1662,60 @@ class RealTimeStreaming:
 
     async def run_realtime_guardrails(
         self,
-        transcript: str,
+        transcript: object,
         item_id: str | None = None,
         pre_block_backend_message: str | None = None,
         event_hooks: Sequence["GuardrailEventHooks"] | None = None,
     ) -> bool:
         """
-        Run registered guardrails on realtime text (transcript, user message, tool output).
-
         Returns True if blocked (synthetic warning already sent to client).
         Returns False if clean (caller should send response.create to the backend).
+        """
+        from litellm.types.guardrails import GuardrailEventHooks
+
+        if isinstance(transcript, str):
+            verdict: Final = await self.apply_realtime_guardrails(
+                (transcript,), "user", item_id, pre_block_backend_message, event_hooks
+            )
+            return isinstance(verdict, RealtimeGuardrailBlock)
+        if realtime_screening_guardrails(
+            "user",
+            event_hooks or (GuardrailEventHooks.realtime_input_transcription,),
+            {**self.request_data, "stream": True},
+            litellm.callbacks,
+        ):
+            raise TypeError(f"Realtime guardrails cannot screen a {type(transcript).__name__} transcript")
+        return False
+
+    async def _screen_client_item(self, guarded_item: RealtimeGuardedItem | None) -> RealtimeGuardrailVerdict:
+        from litellm.types.guardrails import GuardrailEventHooks
+
+        match guarded_item:
+            case RealtimeToolOutput(texts=texts, sanitized_event=sanitized_event):
+                return await self.apply_realtime_guardrails(
+                    texts,
+                    "tool",
+                    pre_block_backend_message=sanitized_event,
+                    event_hooks=[GuardrailEventHooks.pre_call],
+                )
+            case RealtimeMessageText(role=role, texts=texts):
+                return await self.apply_realtime_guardrails(texts, role, event_hooks=[GuardrailEventHooks.pre_call])
+            case None:
+                return None
+
+    async def apply_realtime_guardrails(
+        self,
+        texts: tuple[str, ...],
+        role: str,
+        item_id: str | None = None,
+        pre_block_backend_message: str | None = None,
+        event_hooks: Sequence["GuardrailEventHooks"] | None = None,
+    ) -> RealtimeGuardrailVerdict:
+        """
+        Run registered guardrails on realtime text (transcript, user message, tool output).
+
+        A block has already been reported to the client and the backend when this returns.
+        A rewrite carries the masked texts the caller forwards in place of the originals.
 
         ``pre_block_backend_message`` (if provided) is sent to the backend
         BEFORE any of the guardrail's own backend messages when a block is
@@ -1687,102 +1730,33 @@ class RealTimeStreaming:
         """
         from litellm.types.guardrails import GuardrailEventHooks
 
-        if event_hooks is None:
-            event_hooks = [GuardrailEventHooks.realtime_input_transcription]
-        _realtime_event_types: Final = event_hooks
-        _check_data: Final = {**self.request_data, "transcript": transcript, "stream": True}
-        _already_run: Final[set] = set()
-
-        runner_guardrails: Final = (
-            _realtime_runner_guardrail(callback, _check_data, _realtime_event_types) for callback in litellm.callbacks
+        block: Final = await realtime_guardrail_verdict(
+            texts,
+            role,
+            event_hooks if event_hooks is not None else (GuardrailEventHooks.realtime_input_transcription,),
+            self.request_data,
+            self.user_api_key_dict,
+            litellm.callbacks,
         )
-        for callback in runner_guardrails:
-            if callback is None or id(callback) in _already_run:
-                continue
-            _already_run.add(id(callback))
-            try:
-                await callback.apply_guardrail(
-                    inputs={"texts": [transcript], "images": []},
-                    request_data={"user_api_key_dict": self.user_api_key_dict},
-                    input_type="request",
-                )
-            except Exception as e:
-                # Re-raise unexpected errors (no status_code/detail = programming bug, not a block).
-                # HTTPException and guardrail-raised exceptions have a status_code or detail attr.
-                is_guardrail_block = hasattr(e, "status_code") or isinstance(e, ValueError)
-                if not is_guardrail_block:
-                    verbose_logger.exception(
-                        "[realtime guardrail] unexpected error in apply_guardrail: %s",
-                        e,
-                    )
-                    raise
-                # Extract the human-readable error from the detail dict (HTTPException)
-                # or fall back to str(e) for plain ValueError.
-                detail: object | None = getattr(e, "detail", None)
-                if isinstance(detail, dict):
-                    safe_msg = detail.get("error") or str(e)
-                elif detail is not None:
-                    safe_msg = str(detail)
-                else:
-                    safe_msg = str(e) or "I'm sorry, that request was blocked by the content filter."
+        if not isinstance(block, RealtimeGuardrailBlock):
+            return block
+        await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id)
+        await self.websocket.send_text(guardrail_violation_error_event(block.message))
+        if not self._is_transcription_session:
+            for voice_prompt_event in guardrail_voice_prompt_events(block.message):
+                await self._send_to_backend(voice_prompt_event)
 
-                # Use realtime_violation_message if configured; fall back to guardrail error text.
-                error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg
+        self._violation_count += 1
+        if block.ends_session(self._violation_count):
+            verbose_logger.warning("[realtime guardrail] ending session after violation %d", self._violation_count)
+            await self.backend_ws.close()
 
-                await self._send_blocked_turn_to_backend(pre_block_backend_message, item_id)
-                # Send the policy violation hint (shows as small gray status text in UI).
-                await self.websocket.send_text(
-                    json.dumps(
-                        {
-                            "type": "error",
-                            "error": {
-                                "type": "guardrail_violation",
-                                "message": error_msg,
-                                "code": "content_policy_violation",
-                            },
-                        }
-                    )
-                )
-                if not self._is_transcription_session:
-                    # Ask the LLM to voice the exact guardrail message so the
-                    # user hears it as audio in voice sessions (not just text).
-                    guardrail_prompt = (
-                        f"Say exactly the following message to the user, word for word, "
-                        f"do not add anything else: {error_msg}"
-                    )
-                    await self._send_to_backend(
-                        json.dumps(
-                            {
-                                "type": "conversation.item.create",
-                                "item": {
-                                    "type": "message",
-                                    "role": "user",
-                                    "content": [{"type": "input_text", "text": guardrail_prompt}],
-                                },
-                            }
-                        )
-                    )
-                    await self._send_to_backend(json.dumps({"type": "response.create"}))
-
-                self._violation_count += 1
-                end_session_after: int | None = getattr(callback, "end_session_after_n_fails", None)
-                should_end = getattr(callback, "on_violation", None) == "end_session" or (
-                    end_session_after is not None and self._violation_count >= end_session_after
-                )
-                if should_end:
-                    verbose_logger.warning(
-                        "[realtime guardrail] ending session after violation %d",
-                        self._violation_count,
-                    )
-                    await self.backend_ws.close()
-
-                verbose_logger.warning(
-                    "[realtime guardrail] BLOCKED transcript (violation %d): %r",
-                    self._violation_count,
-                    transcript[:80],
-                )
-                return True
-        return False
+        verbose_logger.warning(
+            "[realtime guardrail] BLOCKED transcript (violation %d): %r",
+            self._violation_count,
+            " ".join(texts)[:80],
+        )
+        return block
 
     async def _handle_provider_config_message(self, raw_response: str) -> None:
         """Process a backend message when a provider_config is set (transformed path)."""
@@ -1866,7 +1840,7 @@ class RealTimeStreaming:
                 self.store_message(event_str)
                 self._capture_transcription_usage(event)
                 await self._send_event_to_client(event, event_str)
-                blocked = await self.run_realtime_guardrails(cast(str, transcript))
+                blocked = await self.run_realtime_guardrails(transcript)
                 if not blocked and self._should_disable_vad_auto_response():
                     await self._send_to_backend(json.dumps({"type": "response.create"}))
                 continue
@@ -2267,92 +2241,22 @@ class RealTimeStreaming:
                 guardrail_turn_detection_injected = False
                 msg_type: str | None = None
                 try:
-                    from litellm.types.guardrails import GuardrailEventHooks
-
                     msg_obj = json.loads(message)
                     client_event = msg_obj
                     msg_type = client_event.get("type")
-
                     if msg_type == "conversation.item.create":
-                        # Check user text messages for prompt injection
-                        item = client_event.get("item", {})
-                        # Check function_call_output first so a client cannot
-                        # bypass the tool-result guardrail by also setting
-                        # role="user" on a function_call_output item.
-                        if item.get("type") == "function_call_output":
-                            # Tool results are client-controlled and fed to the
-                            # model; check them with the same guardrail used for
-                            # user text so an attacker cannot smuggle blocked
-                            # content into a function_call_output.
-                            output: object = item.get("output", "")
-                            output_text = output if isinstance(output, str) else json.dumps(output)
-                            if output_text:
-                                # Build the sanitized function_call_output up
-                                # front so we can hand it to the guardrail
-                                # runner as the pre-block message. Providers
-                                # that pair every toolCall with a toolResponse
-                                # (e.g. Gemini/Vertex Live) require the
-                                # toolResponse to arrive BEFORE any other
-                                # client message — otherwise the guardrail's
-                                # own clientContent would violate the
-                                # pending-tool-call protocol contract and the
-                                # backend could close the connection before
-                                # the sanitized response ever lands. Dropping
-                                # the blocked item outright would similarly
-                                # leave such providers waiting indefinitely.
-                                # The sanitized payload carries no blocked
-                                # content — only a generic policy marker.
-                                sanitized_msg = json.dumps(
-                                    {
-                                        **msg_obj,
-                                        "item": {
-                                            **item,
-                                            "output": json.dumps(
-                                                {
-                                                    "error": "Tool output blocked by content policy",
-                                                }
-                                            ),
-                                        },
-                                    }
-                                )
-                                blocked = await self.run_realtime_guardrails(
-                                    output_text,
-                                    pre_block_backend_message=sanitized_msg,
-                                    event_hooks=[GuardrailEventHooks.pre_call],
-                                )
-                                if blocked:
-                                    # ``_pending_guardrail_message`` is
-                                    # intentionally NOT set here. That flag
-                                    # exists to swallow the reflexive
-                                    # ``response.create`` an OpenAI client
-                                    # sends immediately after a user text
-                                    # message. In a tool-calling flow the
-                                    # client may not send a ``response.create``
-                                    # at all (e.g. Gemini SDKs auto-respond),
-                                    # so leaving the flag set would
-                                    # incorrectly drop an unrelated
-                                    # ``response.create`` from a later
-                                    # interaction turn.
-                                    continue
-                        elif item.get("role") == "user":
-                            content_list: Sequence[object] = item.get("content", [])
-                            texts = [
-                                c.get("text", "")
-                                for c in content_list
-                                if isinstance(c, dict) and c.get("type") == "input_text"
-                            ]
-                            combined_text = " ".join(texts)
-                            if combined_text:
-                                blocked = await self.run_realtime_guardrails(
-                                    combined_text,
-                                    event_hooks=[GuardrailEventHooks.pre_call],
-                                )
-                                if blocked:
-                                    # Store the guardrail reason so the next response.create
-                                    # (sent automatically by the client) is rewritten to
-                                    # include it as response instructions.
-                                    self._pending_guardrail_message = combined_text
-                                    continue  # don't forward the original blocked message
+                        self._pending_guardrail_message = None
+
+                    match await self._screen_client_item(guarded_item := realtime_guarded_item(client_event)):
+                        case RealtimeGuardrailBlock():
+                            if isinstance(guarded_item, RealtimeMessageText):
+                                self._pending_guardrail_message = " ".join(guarded_item.texts)
+                            continue
+                        case RealtimeGuardrailRewrite(texts=masked_texts):
+                            msg_obj["item"] = rewritten_item(client_event, masked_texts)
+                            message = json.dumps(msg_obj)
+                        case None:
+                            pass
 
                     if msg_type == "response.create" and self._pending_guardrail_message:
                         # The guardrail already sent the synthetic AI bubble — drop this

litellm/litellm_core_utils/redact_messages.py

@@ -12,7 +12,7 @@ import copy
 import inspect
 from collections.abc import Iterable, Mapping, Sequence
 from dataclasses import replace
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any, Final, Protocol
 
 from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
 
@@ -21,7 +21,9 @@ from litellm.constants import REDACTED_BY_LITELLM
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.classifier_logging import without_classifier_audit
 from litellm.litellm_core_utils.core_helpers import (
+    attribute_or_none,
     get_metadata_variable_name_from_kwargs,
+    has_attributes,
 )
 from litellm.litellm_core_utils.served_output_texts import SERVED_OUTPUT_TEXTS_KEY
 from litellm.llms.base_llm.guardrail_translation.utils import stream_item_field, stream_item_items
@@ -42,6 +44,11 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObject = Any
 
+
+class _HasArguments(Protocol):
+    arguments: str
+
+
 MESSAGE_REDACTION_ENABLE_HEADERS: Final = ("litellm-enable-message-redaction", "x-litellm-enable-message-redaction")
 
 
@@ -163,7 +170,7 @@ def _redact_bedrock_guardrail_trace(response: object) -> None:
         }
         response.update(traces)
         return
-    for field, value in ((name, getattr(response, name, None)) for name in BEDROCK_GUARDRAIL_TRACE_FIELDS):
+    for field, value in ((name, attribute_or_none(response, name)) for name in BEDROCK_GUARDRAIL_TRACE_FIELDS):
         if value is not None:
             setattr(response, field, _redacted_bedrock_trace(value))
 
@@ -191,8 +198,8 @@ def _redact_tool_calls(tool_calls) -> None:
     if not tool_calls:
         return
     for tool_call in tool_calls:
-        function = getattr(tool_call, "function", None)
-        if function is not None and hasattr(function, "arguments"):
+        function = attribute_or_none(tool_call, "function")
+        if function is not None and has_attributes(function, _HasArguments, "arguments"):
             function.arguments = REDACTED_BY_LITELLM
 
 

litellm/litellm_core_utils/sensitive_data_masker.py

@@ -117,7 +117,9 @@ class SensitiveDataMasker:
             elif key_is_sensitive and isinstance(item, str):
                 masked_items.append(self.mask_value(item))
             else:
-                masked_items.append(item if isinstance(item, (int, float, bool, str, list)) else str(item))
+                masked_items.append(
+                    item if item is None or isinstance(item, (int, float, bool, str, list)) else str(item)
+                )
         return masked_items
 
     def mask_dict(
@@ -144,7 +146,7 @@ class SensitiveDataMasker:
                     str_value = str(v) if v is not None else ""
                     masked_data[k] = self.mask_value(str_value)
                 else:
-                    masked_data[k] = v if isinstance(v, (int, float, bool, str, list)) else str(v)
+                    masked_data[k] = v if v is None or isinstance(v, (int, float, bool, str, list)) else str(v)
             except Exception:
                 masked_data[k] = "<unable to serialize>"
 

litellm/litellm_core_utils/streaming_handler.py

@@ -7,7 +7,7 @@ import logging
 import threading
 import time
 import traceback
-from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence
+from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence, Sized
 from dataclasses import dataclass
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeVar, cast
@@ -17,7 +17,7 @@ import httpx
 from openai.types.chat import ChatCompletionChunk
 from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice
 from openai.types.chat.chat_completion_chunk import ChoiceDelta
-from pydantic import BaseModel, ValidationError
+from pydantic import BaseModel, TypeAdapter, ValidationError
 from typing_extensions import NotRequired, TypedDict
 
 import litellm
@@ -27,7 +27,9 @@ from litellm.constants import STREAM_CHUNK_CHOICE_FIELDS, STREAM_FINISH_CHOICE_F
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.core_helpers import (
     UPSTREAM_USAGE_COST,
+    attribute_or_none,
     chunk_usage,
+    decoded_json_object,
     provider_reported_usage_cost,
     stamp_litellm_bill_on_usage,
 )
@@ -129,6 +131,13 @@ def _apply_provider_fields(response: ModelResponseStream, fields: Mapping[str, o
         setattr(response, key, value)
 
 
+def _entry_len(entries: Mapping[str, object], key: str) -> int:
+    value: Final = entries[key]
+    if isinstance(value, Sized):
+        return len(value)
+    raise TypeError(f"object of type '{type(value).__name__}' has no len()")
+
+
 def _prepend_upstream_choice_extras(
     choice: StreamingChoices, upstream_choice: ChunkChoice, extra_keys: frozenset[str]
 ) -> None:
@@ -346,6 +355,9 @@ class _ParsedChunkHiddenParams(LiteLLMBaseModel):
     upstream_provider: str | None = None
 
 
+_DECODED_JSON: Final = TypeAdapter(object)
+
+
 def _provider_response_model(chunk: object) -> str | None:
     model: Final[object] = chunk.get("model") if isinstance(chunk, Mapping) else getattr(chunk, "model", None)
     return model if isinstance(model, str) and model else None
@@ -432,9 +444,8 @@ class CustomStreamWrapper:
         self._closed = False
         self._stream_created_time: float = time.time()
 
-        litellm_params: Final[GenericLiteLLMParams] = GenericLiteLLMParams.model_validate(
-            dict(**self.logging_obj.model_call_details.get("litellm_params", {}))
-        )
+        raw_litellm_params: Final = dict[str, object](**self.logging_obj.model_call_details.get("litellm_params", {}))
+        litellm_params: Final[GenericLiteLLMParams] = GenericLiteLLMParams.model_validate(raw_litellm_params)
         self.merge_reasoning_content_in_choices: bool = litellm_params.merge_reasoning_content_in_choices or False
         self.sent_first_thinking_block = False
         self.sent_last_thinking_block = False
@@ -465,7 +476,7 @@ class CustomStreamWrapper:
 
         _api_base: Final = get_api_base(
             model=model or "",
-            optional_params=self.logging_obj.model_call_details.get("litellm_params", {}),
+            optional_params=raw_litellm_params,
         )
 
         self._hidden_params: dict[str, object] = {
@@ -953,7 +964,7 @@ class CustomStreamWrapper:
                 if "text_output" in chunk:
                     response = CustomStreamWrapper.strip_sse_data_from_chunk(chunk) or ""
                     response = response.strip()
-                    parsed_response = json.loads(response)
+                    parsed_response = decoded_json_object(_DECODED_JSON.validate_python(json.loads(response)))
                 else:
                     return {
                         "text": "",
@@ -1087,7 +1098,7 @@ class CustomStreamWrapper:
 
     def is_chunk_non_empty(
         self,
-        completion_obj: dict[str, Any],
+        completion_obj: Mapping[str, object],
         model_response: ModelResponseStream,
         response_obj: Mapping[str, object],
     ) -> bool:
@@ -1097,7 +1108,7 @@ class CustomStreamWrapper:
             or (
                 "tool_calls" in completion_obj
                 and completion_obj["tool_calls"] is not None
-                and len(completion_obj["tool_calls"]) > 0
+                and _entry_len(completion_obj, "tool_calls") > 0
             )
             or ("function_call" in completion_obj and completion_obj["function_call"] is not None)
             or (
@@ -1850,7 +1861,9 @@ class CustomStreamWrapper:
                             elif delta.tool_calls is not None:
                                 if isinstance(delta.tool_calls, list):
                                     for t in delta.tool_calls:
-                                        if hasattr(t, "functions") and hasattr(getattr(t, "functions"), "arguments"):
+                                        if hasattr(t, "functions") and hasattr(
+                                            attribute_or_none(t, "functions"), "arguments"
+                                        ):
                                             if t.function.arguments is None:
                                                 t.function.arguments = ""
                             _json_delta: Final[_DeltaDumpDict] = delta.model_dump()
@@ -1888,7 +1901,7 @@ class CustomStreamWrapper:
                     return
             ## CHECK FOR TOOL USE
 
-            if "tool_calls" in completion_obj and len(completion_obj["tool_calls"]) > 0:
+            if "tool_calls" in completion_obj and _entry_len(completion_obj, "tool_calls") > 0:
                 if self.is_function_call is True:  # user passed in 'functions' param
                     completion_obj["function_call"] = completion_obj["tool_calls"][0]["function"]
                     completion_obj["tool_calls"] = None
@@ -2817,8 +2830,9 @@ class CustomStreamWrapper:
 
     def _record_and_strip_usage_chunk(self, chunk: ModelResponseStream) -> ModelResponseStream | None:
         self.chunks.append(chunk.model_copy())
+        dumped_chunk: Final = MappingProxyType[str, object](chunk.model_dump())
         usage_free_chunk: Final = self.model_response_creator(
-            chunk={key: value for key, value in chunk.model_dump().items() if key != "usage"},
+            chunk={key: value for key, value in dumped_chunk.items() if key != "usage"},
             hidden_params=cast(  # cast-ok: preserve mapping operations on dynamic storage
                 Mapping[str, object], getattr(chunk, HIDDEN_PARAMS_ATTR)
             ),

litellm/llms/a2a/chat/guardrail_translation/handler.py

@@ -11,13 +11,15 @@ A2A Protocol Format:
 """
 
 import json
-from collections.abc import Sequence
-from typing import TYPE_CHECKING, Any, Final, Optional
+from collections.abc import Mapping, Sequence
+from functools import reduce
+from typing import TYPE_CHECKING, Any, Final, Optional, Protocol, runtime_checkable
 
 from pydantic import ConfigDict, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
+from litellm.litellm_core_utils.core_helpers import has_attributes, require_attributes, subscript
 from litellm.llms.base_llm.guardrail_translation.base_translation import (
     BaseTranslation,
     StreamingScanKey,
@@ -30,6 +32,38 @@ if TYPE_CHECKING:
     from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
 
 _JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True))
+_UNSUPPORTED_RESPONSE: Final = object()
+
+
+class _ModelDumpable(Protocol):
+    def model_dump(self) -> object: ...
+
+
+class _ResultLookup(Protocol):
+    def get(self, key: str, default: object, /) -> object: ...
+
+
+@runtime_checkable
+class _SupportsSetItem(Protocol):
+    def __setitem__(self, key: str, value: object, /) -> None: ...
+
+
+def _item_assignment_target(value: object) -> _SupportsSetItem:
+    if isinstance(value, _SupportsSetItem):
+        return value
+    raise TypeError(f"'{type(value).__name__}' object does not support item assignment")
+
+
+def _response_payload(response: object) -> object:
+    if has_attributes(response, _ModelDumpable, "model_dump"):
+        return response.model_dump()
+    if isinstance(response, dict):
+        return response
+    return _UNSUPPORTED_RESPONSE
+
+
+def _path_step(current: object, key: str) -> object:
+    return subscript(current, int(key) if key.isdigit() else key)
 
 
 class _A2ATextPart(TypedDict, total=False):
@@ -123,7 +157,7 @@ class A2AGuardrailHandler(BaseTranslation):
 
     async def process_output_response(
         self,
-        response: Any,
+        response: object,
         guardrail_to_apply: "CustomGuardrail",
         litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None,
         user_api_key_dict: Optional["UserAPIKeyAuth"] = None,
@@ -147,17 +181,12 @@ class A2AGuardrailHandler(BaseTranslation):
         Returns:
             Modified response with guardrails applied to text content
         """
-        # Handle both dict and Pydantic model responses
-        if hasattr(response, "model_dump"):
-            response_dict = response.model_dump()
-            is_pydantic = True
-        elif isinstance(response, dict):
-            response_dict = response
-            is_pydantic = False
-        else:
+        payload: Final = _response_payload(response)
+        if payload is _UNSUPPORTED_RESPONSE:
             verbose_proxy_logger.warning("A2A: Unknown response type %s, skipping guardrail", type(response))
             return response
 
+        response_dict: Final = require_attributes(payload, _ResultLookup, "get")
         result: Final = response_dict.get("result", {})
         if not result or not isinstance(result, dict):
             verbose_proxy_logger.debug("A2A: No result in response, skipping guardrail")
@@ -218,15 +247,8 @@ class A2AGuardrailHandler(BaseTranslation):
 
         verbose_proxy_logger.debug("A2A: Processed output response")
 
-        # Update the original response
-        if is_pydantic:
-            # For Pydantic models, we need to update the underlying dict
-            # and the model will reflect the changes
-            response_dict["result"] = result
-            return response
-        else:
-            response["result"] = result
-            return response
+        _item_assignment_target(response_dict)["result"] = result
+        return response
 
     async def process_output_streaming_response(
         self,
@@ -448,20 +470,11 @@ class A2AGuardrailHandler(BaseTranslation):
 
     def _apply_text_to_path(
         self,
-        result: dict[str | int, Any],
+        result: Mapping[str, object],
         path: tuple[str, ...],
         part_idx: int,
         text: str,
     ) -> None:
         """Apply guardrailed text back to the specified path in the result."""
-        # Navigate to the parts list
-        current = result
-        for key in path:
-            if key.isdigit():
-                # Array index
-                current = current[int(key)]
-            else:
-                current = current[key]
-
-        # Update the text in the part
-        current[part_idx]["text"] = text
+        parts: Final = reduce(_path_step, path, result)
+        _item_assignment_target(subscript(parts, part_idx))["text"] = text

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -795,18 +795,21 @@ class AnthropicMessagesHandler(BaseTranslation):
         data: dict,  # mutable-ok: the request body the adapter reads
         *,
         keep_cache_control: bool = False,
+        keep_tool_errors: bool = False,
     ) -> ChatCompletionRequest:
         """Translate Anthropic request to OpenAI chat completion format."""
         (
             chat_completion_compatible_request,
             _tool_name_mapping,
-        ) = LiteLLMAnthropicMessagesAdapter(keep_cache_control=keep_cache_control).translate_anthropic_to_openai(
+        ) = LiteLLMAnthropicMessagesAdapter(
+            keep_cache_control=keep_cache_control, keep_tool_errors=keep_tool_errors
+        ).translate_anthropic_to_openai(
             anthropic_message_request=cast(AnthropicMessagesRequest, scannable_tool_result_request(data)),
             preserve_midturn_system=True,
         )
         return chat_completion_compatible_request
 
-    def get_structured_messages(self, data: dict) -> list[AllMessageValues] | None:
+    def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list[AllMessageValues] | None:
         """
         Convert Anthropic messages request data to OpenAI-spec structured messages.
 
@@ -815,7 +818,7 @@ class AnthropicMessagesHandler(BaseTranslation):
         messages: Final = data.get("messages")
         if messages is None:
             return None
-        chat_completion_compatible_request: Final = self._translate_to_openai(data)
+        chat_completion_compatible_request: Final = self._translate_to_openai(data, keep_tool_errors=keep_tool_errors)
         result: Final = cast(
             list[AllMessageValues],
             chat_completion_compatible_request.get("messages", []),

litellm/llms/anthropic/chat/handler.py

@@ -1081,7 +1081,7 @@ class ModelResponseIterator:
                     if "caller" in content_block_start["content_block"]:
                         caller_data: Final = content_block_start["content_block"]["caller"]
                         if caller_data and tool_use is not None:
-                            tool_use["caller"] = cast(dict[str, Any], caller_data)
+                            tool_use["caller"] = caller_data
                 elif content_block_start["content_block"]["type"] == "redacted_thinking":
                     (
                         thinking_blocks,

litellm/llms/anthropic/common_utils.py

@@ -2107,7 +2107,7 @@ def _strip_empty_content_blocks_from_message(message: object) -> object:
 
 def strip_empty_content_blocks_from_anthropic_messages(
     messages: Sequence[object],
-) -> list[Any]:
+) -> Sequence[object]:
     """
     Return a new message list with empty or whitespace-only ``{"type": "text"}``
     content blocks and empty *unsigned* ``{"type": "thinking"}`` blocks removed.
@@ -2220,7 +2220,7 @@ def _sanitize_tool_use_id_content_block(block: object) -> object:
     return block
 
 
-def sanitize_tool_use_ids_in_anthropic_messages(messages: Sequence[object]) -> list[Any]:
+def sanitize_tool_use_ids_in_anthropic_messages(messages: Sequence[object]) -> Sequence[object]:
     """
     Return a new message list with ``tool_use`` / ``server_tool_use`` ``id`` and
     ``tool_result`` ``tool_use_id`` values rewritten to satisfy Anthropic's

litellm/llms/anthropic/messages/adapters/transformation.py

@@ -8,7 +8,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, TypeVar, cast
 from pydantic import JsonValue, TypeAdapter
 
 import litellm
-from litellm.litellm_core_utils.core_helpers import normalize_drop_params
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, normalize_drop_params
 from litellm.litellm_core_utils.hidden_params import get_hidden_params
 from litellm.llms.anthropic.messages.convertible_blocks import tool_result_text_part
 from litellm.llms.anthropic.messages.utils import (
@@ -434,8 +434,9 @@ _CLAUDE_OVER_REASONING_EFFORT_PROVIDERS: Final = frozenset({LlmProviders.KYMA.va
 
 
 class LiteLLMAnthropicMessagesAdapter:
-    def __init__(self, *, keep_cache_control: bool = False) -> None:
+    def __init__(self, *, keep_cache_control: bool = False, keep_tool_errors: bool = False) -> None:
         self.keep_cache_control: Final = keep_cache_control
+        self.keep_tool_errors: Final = keep_tool_errors
 
     ### FOR [BETA] `/v1/messages` endpoint support
 
@@ -615,11 +616,7 @@ class LiteLLMAnthropicMessagesAdapter:
                                 self._add_cache_control_if_applicable(content, doc_obj, model)
                                 new_user_content_list.append(doc_obj)
                         elif content.get("type") == "tool_result":
-                            tool_result = ChatCompletionToolMessage(
-                                role="tool",
-                                tool_call_id=content.get("tool_use_id", ""),
-                                content=self._tool_result_content(content.get("content")),
-                            )
+                            tool_result = self._tool_message(content)
                             self._add_cache_control_if_applicable(content, tool_result, model)
                             tool_message_list.append(tool_result)
 
@@ -1432,6 +1429,15 @@ class LiteLLMAnthropicMessagesAdapter:
             return None
         return anthropic_image_source_to_openai_url(image_source)
 
+    def _tool_message(self, block: object) -> ChatCompletionToolMessage:
+        fields: Final = as_str_mapping(block) or {}
+        is_error: Final = fields.get("is_error")
+        tool_call_id: Final = tool_use_id if isinstance(tool_use_id := fields.get("tool_use_id"), str) else ""
+        content: Final = self._tool_result_content(fields.get("content"))
+        if self.keep_tool_errors and isinstance(is_error, bool):
+            return ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content, is_error=is_error)
+        return ChatCompletionToolMessage(role="tool", tool_call_id=tool_call_id, content=content)
+
     def _tool_result_content(self, raw_content: object) -> ToolResultContent:
         if isinstance(raw_content, str):
             return raw_content

litellm/llms/anthropic/messages/native/agentic_streaming_iterator.py

@@ -17,11 +17,14 @@ execute it.
 import asyncio
 import contextlib
 import json
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterable, AsyncIterator
 from typing import TYPE_CHECKING, Any, Final, cast
 
+from pydantic import TypeAdapter
+
 from litellm._logging import verbose_logger
 from litellm.constants import STREAM_SSE_KEEPALIVE_PING_BYTES
+from litellm.litellm_core_utils.core_helpers import decoded_json_object, has_attributes
 from litellm.types.integrations.custom_logger import AgenticLoopLimitError, AgenticLoopSafetyError
 
 if TYPE_CHECKING:
@@ -42,6 +45,7 @@ AGENTIC_LOOP_LIMIT_ERROR_SSE_BYTES: Final = (
     b'"The agentic loop reached its limit before this turn finished. '
     b'Raise max_agentic_loops to allow more loops."}}\n\n'
 )
+_DECODED_JSON: Final = TypeAdapter(object)
 
 
 def is_server_fulfilled_tool_leak_error(chunk: object) -> bool:
@@ -76,10 +80,10 @@ def _parse_sse_events(raw: bytes) -> list[tuple]:
             continue
         data_str = stripped[len("data:") :].strip()
         try:
-            data = json.loads(data_str)
+            data = _DECODED_JSON.validate_python(json.loads(data_str))
         except (json.JSONDecodeError, ValueError):
             continue
-        event_type = current_event_type or data.get("type", "")
+        event_type = current_event_type or decoded_json_object(data).get("type", "")
         current_event_type = None
         events.append((event_type, data))
     return events
@@ -371,7 +375,7 @@ class AgenticAnthropicStreamingIterator:
                 verbose_logger.debug("AgenticStreamingIterator: Could not rebuild response from SSE bytes")
                 return
 
-            result: Final = await self._http_handler._call_agentic_completion_hooks(
+            result: Final[object] = await self._http_handler._call_agentic_completion_hooks(
                 response=rebuilt,
                 model=self._model,
                 messages=self._messages,
@@ -386,7 +390,7 @@ class AgenticAnthropicStreamingIterator:
             if result is None:
                 return
 
-            if hasattr(result, "__aiter__"):
+            if has_attributes(result, AsyncIterable[object], "__aiter__"):
                 self._follow_up_iterator = result.__aiter__()
             elif isinstance(result, dict):
                 from litellm.llms.anthropic.messages.native.fake_stream_iterator import (

litellm/llms/anthropic/messages/responses_adapters/streaming_iterator.py

@@ -2,8 +2,8 @@
 ## Translates OpenAI call to Anthropic `/v1/messages` format
 import asyncio
 from collections import deque
-from collections.abc import AsyncGenerator, Awaitable, Callable, Iterator, Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final
+from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
+from typing import TYPE_CHECKING, Final
 
 import anyio
 from pydantic import ConfigDict, field_validator
@@ -12,6 +12,7 @@ from litellm import verbose_logger
 from litellm._logging import redact_internal_details_from_client_message
 from litellm._uuid import uuid
 from litellm.exceptions import APIError, MidStreamFallbackError
+from litellm.litellm_core_utils.core_helpers import has_attributes, iter_items
 from litellm.litellm_core_utils.prompt_templates.reasoning_utils import (
     encrypted_reasoning_signature,
 )
@@ -126,7 +127,7 @@ class AnthropicResponsesStreamWrapper:
 
     def __init__(
         self,
-        responses_stream: Any,
+        responses_stream: object,
         model: str,
         litellm_logging_obj: "LiteLLMLoggingObject | None" = None,
         safeguards_evaluator: "SafeguardsEvaluator | None" = None,
@@ -414,7 +415,9 @@ class AnthropicResponsesStreamWrapper:
             response_obj: Final = getattr(event, "response", None) or (
                 event.get("response") if isinstance(event, dict) else None
             )
-            output: Final = (getattr(response_obj, "output", None) or ()) if response_obj is not None else ()
+            output: Final[Iterable[object]] = (
+                (getattr(response_obj, "output", None) or ()) if response_obj is not None else ()
+            )
             refusal_text: Final = responses_output_refusal_text(output) or (self._refusal_text or None)
             status: Final = getattr(response_obj, "status", None) if response_obj is not None else None
             has_tool_call: Final = any(
@@ -508,14 +511,14 @@ class AnthropicResponsesStreamWrapper:
             return self._chunk_queue.popleft()
 
         try:
-            if hasattr(self.responses_stream, "__aiter__"):
+            if has_attributes(self.responses_stream, AsyncIterable[object], "__aiter__"):
                 async for event in self.responses_stream:
                     self._process_event(event)
                     if self._chunk_queue:
                         return self._chunk_queue.popleft()
             else:
                 if self._sync_responses_iterator is None:
-                    self._sync_responses_iterator = iter(self.responses_stream)
+                    self._sync_responses_iterator = iter(iter_items(self.responses_stream))
                 sync_iterator: Final = self._sync_responses_iterator
                 missing: Final = object()
                 while (event := await asyncio.to_thread(next, sync_iterator, missing)) is not missing:

litellm/llms/anthropic/messages/responses_adapters/transformation.py

@@ -10,6 +10,8 @@ from collections.abc import Iterable, Mapping, Sequence
 from itertools import groupby
 from typing import Any, Final, cast
 
+from pydantic import TypeAdapter
+
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     TOOL_RESULT_IMAGE_BOUNDARY,
     TOOL_RESULT_IMAGE_PLACEHOLDER,
@@ -58,6 +60,7 @@ from litellm.types.llms.openai import (
 
 REASONING_SUMMARY_PART_SEPARATOR: Final = "\n\n"
 RESPONSES_INCLUDE_ENCRYPTED_REASONING: Final = "reasoning.encrypted_content"
+_DECODED_JSON: Final = TypeAdapter(object)
 
 
 class LiteLLMAnthropicToResponsesAPIAdapter:
@@ -805,15 +808,17 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
                         content.append(reasoning_block)
                 elif item_type == "function_call":
                     try:
-                        input_data = json.loads(item.get("arguments", "{}"))
+                        input_data = _DECODED_JSON.validate_python(json.loads(item.get("arguments", "{}")))
                     except (json.JSONDecodeError, TypeError):
                         input_data = {}
                     content.append(
-                        AnthropicResponseContentBlockToolUse(
-                            type="tool_use",
-                            id=item.get("call_id") or item.get("id", ""),
-                            name=item.get("name", ""),
-                            input=input_data,
+                        AnthropicResponseContentBlockToolUse.model_validate(
+                            {
+                                "type": "tool_use",
+                                "id": item.get("call_id") or item.get("id", ""),
+                                "name": item.get("name", ""),
+                                "input": input_data,
+                            }
                         ).model_dump(exclude_none=True)
                     )
                     stop_reason = "tool_use"

litellm/llms/anthropic/messages/stream_assembly.py

@@ -4,10 +4,13 @@ from collections.abc import Mapping, Sequence
 from types import MappingProxyType
 from typing import Final
 
+from pydantic import TypeAdapter
+
 import litellm
 from litellm._logging import verbose_proxy_logger
-from litellm.litellm_core_utils.core_helpers import as_str_mapping, map_finish_reason
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, map_finish_reason
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import is_str_keyed_dict
 from litellm.llms.anthropic.chat.handler import (
     ModelResponseIterator as AnthropicModelResponseIterator,
 )
@@ -21,6 +24,8 @@ from litellm.types.utils import (
     TextCompletionResponse,
 )
 
+_DECODED_JSON: Final = TypeAdapter(object)
+
 
 def split_sse_chunk_into_events(chunk: str | bytes) -> list[str]:
     """
@@ -98,6 +103,14 @@ _FAST_PATH_DISALLOWED_DELTA_TYPES: Final = frozenset(
 )
 
 
+def _joined_text(parts: Sequence[object]) -> str:
+    strings: Final = [part for part in parts if isinstance(part, str)]
+    if len(strings) == len(parts):
+        return "".join(strings)
+    first_bad: Final = next(position for position, part in enumerate(parts) if not isinstance(part, str))
+    raise TypeError(f"sequence item {first_bad}: expected str instance, {type(parts[first_bad]).__name__} found")
+
+
 def collapse_pure_text_chunks(
     all_chunks: Sequence[str | bytes],
 ) -> list[str] | None:
@@ -120,10 +133,10 @@ def collapse_pure_text_chunks(
             if ev:
                 normalized.append(ev)
 
-    text_block_indexes: Final[set] = set()
+    text_block_indexes: Final[set[object]] = set()
     out: Final[list[str]] = []
-    pending_text: list[str] = []
-    pending_index: int | None = None
+    pending_text: list[object] = []
+    pending_index: object = None
     saw_any_text_delta = False
 
     def flush() -> None:
@@ -132,7 +145,7 @@ def collapse_pure_text_chunks(
             merged: Final = {
                 "type": "content_block_delta",
                 "index": pending_index if pending_index is not None else 0,
-                "delta": {"type": "text_delta", "text": "".join(pending_text)},
+                "delta": {"type": "text_delta", "text": _joined_text(pending_text)},
             }
             out.append("data: " + json.dumps(merged))
             pending_text = []
@@ -158,20 +171,20 @@ def collapse_pure_text_chunks(
         if not json_str.startswith("{"):
             continue
         try:
-            data = json.loads(json_str)
+            data = decoded_json_object(_DECODED_JSON.validate_python(json.loads(json_str)))
         except (json.JSONDecodeError, ValueError):
             return None
 
         etype = data.get("type")
         if etype == "content_block_start":
-            block = data.get("content_block") or {}
+            block = decoded_json_object(data.get("content_block") or {})
             if block.get("type") != "text":
                 return None
             text_block_indexes.add(data.get("index"))
             flush()
             out.append(ev)
         elif etype == "content_block_delta":
-            delta = data.get("delta") or {}
+            delta = decoded_json_object(data.get("delta") or {})
             dtype = delta.get("type")
             if dtype in _FAST_PATH_DISALLOWED_DELTA_TYPES:
                 return None
@@ -291,9 +304,10 @@ def extract_sse_data(event_str: str) -> dict | None:
     if payload is None or not payload.startswith("{"):
         return None
     try:
-        return json.loads(payload)
+        decoded: Final = _DECODED_JSON.validate_python(json.loads(payload))
     except (ValueError, TypeError):
         return None
+    return decoded if is_str_keyed_dict(decoded) else None
 
 
 def build_usage_only_response_from_chunks(
@@ -492,7 +506,7 @@ def _carries_no_data(fields: Sequence[tuple[str, str]]) -> bool:
 
 def _json_mapping(payload: str) -> Mapping[str, object] | None:
     try:
-        value: Final[object] = json.loads(payload)
+        value: Final = _DECODED_JSON.validate_python(json.loads(payload))
     except ValueError:
         return None
     return as_str_mapping(value)
@@ -555,7 +569,7 @@ def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
 
 def _is_json_no_reader_can_take(payload: str) -> bool:
     try:
-        value: Final[object] = json.loads(payload)
+        value: Final = _DECODED_JSON.validate_python(json.loads(payload))
     except ValueError:
         return False
     if value is None:

litellm/llms/apiserpent/search/transformation.py

@@ -12,7 +12,9 @@ from typing import Final, Literal, cast
 from urllib.parse import urlencode
 
 import httpx
+from pydantic import TypeAdapter
 
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, decoded_json_object, iter_items
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.url_utils import ensure_api_route
 from litellm.llms.apiserpent.search.defaults import (
@@ -33,6 +35,7 @@ from litellm.secret_managers.main import get_secret_str
 DEEP_SEARCH_PARAM: Final = "deep"
 APISERPENT_BASE: Final = "https://apiserpent.com"
 APISERPENT_PARAMS_KEY: Final = "_apiserpent_params"
+_DECODED_JSON: Final = TypeAdapter(object)
 
 
 class APISerpentSearchConfig(BaseSearchConfig):
@@ -150,22 +153,24 @@ class APISerpentSearchConfig(BaseSearchConfig):
         Full format nests results under ``results.organic[]``; simple format
         returns a flat ``results[]`` array. Both expose title/url/snippet.
         """
-        response_json: Final = raw_response.json()
+        response_json: Final = decoded_json_object(_DECODED_JSON.validate_python(raw_response.json()))
 
         raw_results: Final = response_json.get("results") or {}
-        organic: Final = raw_results.get("organic", []) if isinstance(raw_results, dict) else raw_results
+        nested_results: Final = as_str_mapping(raw_results)
+        organic: Final = nested_results.get("organic", []) if nested_results is not None else raw_results
 
-        results: Final[list[SearchResult]] = []
-        for result in organic:
-            results.append(
-                SearchResult(
-                    title=result.get("title", ""),
-                    url=result.get("url", ""),
-                    snippet=result.get("snippet", ""),
-                    date=result.get("date"),
-                    last_updated=None,
-                )
+        results: Final = [
+            SearchResult.model_validate(
+                {
+                    "title": result.get("title", ""),
+                    "url": result.get("url", ""),
+                    "snippet": result.get("snippet", ""),
+                    "date": result.get("date"),
+                    "last_updated": None,
+                }
             )
+            for result in map(decoded_json_object, iter_items(organic))
+        ]
 
         return SearchResponse(
             results=results,

litellm/llms/azure/azure.py

@@ -2,7 +2,7 @@ import asyncio
 import json
 import time
 from collections.abc import Callable, Coroutine
-from typing import Final
+from typing import Final, Protocol
 
 import httpx
 from openai import (
@@ -17,6 +17,7 @@ from pydantic import BaseModel
 
 import litellm
 from litellm.constants import DEFAULT_MAX_RETRIES
+from litellm.litellm_core_utils.core_helpers import require_attributes
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.logging_utils import speech_request_body, track_llm_api_timing
 from litellm.llms.custom_httpx.http_handler import (
@@ -60,6 +61,11 @@ from .image_generation import (
 from .image_generation.http_utils import azure_deployment_image_generation_json_body
 
 
+class _HasResponseText(Protocol):
+    @property
+    def text(self) -> str: ...
+
+
 class AzureOpenAIAssistantsAPIConfig:
     """
     Reference: https://learn.microsoft.com/en-us/azure/ai-services/openai/assistants-reference-messages?tabs=python#create-message
@@ -853,11 +859,11 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         except Exception as e:
             status_code: Final = getattr(e, "status_code", 500)
             error_headers = getattr(e, "headers", None)
-            error_response: Final = getattr(e, "response", None)
+            error_response: Final[object] = getattr(e, "response", None)
             error_text = str(e)
             if error_headers is None and error_response:
                 error_headers = getattr(error_response, "headers", None)
-                error_text = error_response.text
+                error_text = require_attributes(error_response, _HasResponseText, "text").text
             raise AzureOpenAIError(
                 status_code=status_code,
                 message=error_text,

litellm/llms/azure/responses/transformation.py

@@ -251,9 +251,8 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
 
     def compact_url(self, api_base: str) -> str:
         params: Final = httpx.URL(api_base).params
-        v1_params: Final = (
-            params if BaseAzureLLM.is_azure_v1_api_version(params.get("api-version")) else params.remove("api-version")
-        )
+        api_version: Final = next(iter(params.get_list("api-version")), None)
+        v1_params: Final = params if BaseAzureLLM.is_azure_v1_api_version(api_version) else params.remove("api-version")
         return super().compact_url(str(_v1_responses_url(api_base).copy_with(params=v1_params)))
 
     #########################################################

litellm/llms/azure_ai/agents/handler.py

@@ -308,7 +308,7 @@ class AzureAIAgentsHandler:
         logging_obj: LiteLLMLoggingObj,
         optional_params: dict,
         litellm_params: dict,
-        timeout: float,
+        timeout: float | str | httpx.Timeout | None,
         client: HTTPHandler | None = None,
         headers: dict | None = None,
     ) -> ModelResponse:
@@ -432,7 +432,7 @@ class AzureAIAgentsHandler:
         logging_obj: LiteLLMLoggingObj,
         optional_params: dict,
         litellm_params: dict,
-        timeout: float,
+        timeout: float | str | httpx.Timeout | None,
         client: AsyncHTTPHandler | None = None,
         headers: dict | None = None,
     ) -> ModelResponse:
@@ -559,7 +559,7 @@ class AzureAIAgentsHandler:
         logging_obj: LiteLLMLoggingObj,
         optional_params: dict,
         litellm_params: dict,
-        timeout: float,
+        timeout: float | str | httpx.Timeout | None,
         headers: dict | None = None,
     ) -> AsyncIterator[ModelResponseStream]:
         """Execute async streaming completion using Azure Agent Service with native SSE."""

litellm/llms/azure_ai/agents/transformation.py

@@ -323,7 +323,7 @@ class AzureAIAgentsConfig(BaseConfig):
         logging_obj: LiteLLMLoggingObj,
         optional_params: dict,
         litellm_params: dict,
-        timeout: float | Any,
+        timeout: float | str | httpx.Timeout | None,
         acompletion: bool,
         stream: bool | None = False,
         headers: dict | None = None,

litellm/llms/azure_ai/anthropic/count_tokens/handler.py

@@ -8,7 +8,7 @@ from collections.abc import Mapping
 from typing import Final
 
 import httpx
-from pydantic import JsonValue
+from pydantic import JsonValue, TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
@@ -22,6 +22,8 @@ from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
 )
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 
+_DECODED_JSON: Final = TypeAdapter(object)
+
 
 class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
     """
@@ -42,7 +44,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
         system: object = None,
         optional_params: Mapping[str, JsonValue] | None = None,
         context_management: JsonValue = None,
-    ) -> dict[str, object]:
+    ) -> object:
         """
         Handle a CountTokens request using httpx with Azure authentication.
 
@@ -126,7 +128,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
                     message=error_text,
                 )
 
-            azure_response: Final = response.json()
+            azure_response: Final = _DECODED_JSON.validate_python(response.json())
 
             verbose_logger.debug("Azure AI Anthropic response: %s", azure_response)
 

litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py

@@ -11,6 +11,7 @@ from pydantic import JsonValue
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import decoded_json_object
 from litellm.llms.anthropic.count_tokens.transformation import (
     AnthropicCountFields,
     ChatCountTranslationError,
@@ -139,13 +140,14 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
             )
 
             if result is not None:
+                response_fields: Final = decoded_json_object(result)
                 return TokenCountResponse.model_validate(
                     {
-                        "total_tokens": result.get("input_tokens", 0),
+                        "total_tokens": response_fields.get("input_tokens", 0),
                         "request_model": request_model,
                         "model_used": model_to_use,
                         "tokenizer_type": "azure_ai_anthropic_api",
-                        "original_response": result,
+                        "original_response": response_fields,
                     }
                 )
         except AnthropicError as e:

litellm/llms/azure_ai/anthropic/messages_transformation.py

@@ -3,7 +3,7 @@ Azure Anthropic messages transformation config - extends AnthropicMessagesConfig
 """
 
 from collections.abc import Sequence
-from typing import Any, Final
+from typing import Final
 
 from litellm.llms.anthropic.messages.native.transformation import (
     AnthropicMessagesConfig,
@@ -109,14 +109,10 @@ class AzureAnthropicMessagesConfig(AnthropicMessagesConfig):
         Processes both `system` and `messages` content blocks.
         """
 
-        def _sanitize(cache_control: Any) -> None:
-            if isinstance(cache_control, dict):
-                cache_control.pop("scope", None)
-
         def _process_content_list(content: list) -> None:
             for item in content:
-                if isinstance(item, dict) and "cache_control" in item:
-                    _sanitize(item["cache_control"])
+                if isinstance(item, dict) and "cache_control" in item and isinstance(item["cache_control"], dict):
+                    item["cache_control"].pop("scope", None)
 
         if "system" in anthropic_messages_request:
             system: Final = anthropic_messages_request["system"]

litellm/llms/azure_ai/image_edit/flux2_transformation.py

@@ -2,11 +2,12 @@ import base64
 from collections.abc import Mapping, Sequence
 from io import BufferedReader
 from types import MappingProxyType
-from typing import Any, Final
+from typing import Final, Protocol
 
 from httpx._types import RequestFiles
 
 import litellm
+from litellm.litellm_core_utils.core_helpers import has_attributes
 from litellm.llms.azure_ai.common_utils import (
     AzureFoundryModelInfo,
     get_azure_ai_auth_headers,
@@ -21,6 +22,10 @@ from litellm.types.llms.openai import FileTypes
 from litellm.types.router import GenericLiteLLMParams
 
 
+class _BinaryReadable(Protocol):
+    def read(self) -> bytes: ...
+
+
 class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
     """
     Azure AI Foundry FLUX 2 image edit config
@@ -100,7 +105,7 @@ class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
         if image is None:
             raise ValueError("FLUX 2 image edit requires an image.")
 
-        images: Final = tuple(image) if isinstance(image, list) else (image,)
+        images: Final[Sequence[object]] = [*image] if isinstance(image, list) else [image]
         if not images:
             raise ValueError("FLUX 2 image edit requires at least one image.")
         max_reference_images: Final = 10 if "flex" in model.lower() else 8
@@ -121,14 +126,14 @@ class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
         }
         return request_body, []
 
-    def _convert_image_to_base64(self, image: Any) -> str:
+    def _convert_image_to_base64(self, image: object) -> str:
         """Convert image file to base64 string"""
         if isinstance(image, BufferedReader):
             image_bytes = image.read()
             image.seek(0)  # Reset file pointer for potential reuse
         elif isinstance(image, bytes):
             image_bytes = image
-        elif hasattr(image, "read"):
+        elif has_attributes(image, _BinaryReadable, "read"):
             image_bytes = image.read()
         else:
             raise ValueError(f"Unsupported image type: {type(image)}")

litellm/llms/base_llm/guardrail_translation/base_translation.py

@@ -1,7 +1,9 @@
 from abc import ABC, abstractmethod
 from collections.abc import Sequence
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Optional, Protocol
+
+from litellm.litellm_core_utils.core_helpers import has_attributes
 
 if TYPE_CHECKING:
     from fastapi import HTTPException
@@ -15,6 +17,10 @@ if TYPE_CHECKING:
     from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth
 
 
+class _ModelDumpable(Protocol):
+    def model_dump(self) -> object: ...
+
+
 @dataclass(slots=True)
 class StreamTransformSink:
     """Out-parameter used by ``process_output_streaming_response`` to hand the
@@ -81,7 +87,7 @@ class BaseTranslation(ABC):
 
     @staticmethod
     def transform_user_api_key_dict_to_metadata(
-        user_api_key_dict: Any | None,
+        user_api_key_dict: object,
     ) -> dict[str, object]:
         """
         Transform user_api_key_dict to a metadata dict with prefixed keys.
@@ -99,7 +105,11 @@ class BaseTranslation(ABC):
             return {}
 
         # Convert to dict if it's a Pydantic object
-        user_dict = user_api_key_dict.model_dump() if hasattr(user_api_key_dict, "model_dump") else user_api_key_dict
+        user_dict: Final = (
+            user_api_key_dict.model_dump()
+            if has_attributes(user_api_key_dict, _ModelDumpable, "model_dump")
+            else user_api_key_dict
+        )
 
         if not isinstance(user_dict, dict):
             return {}
@@ -257,11 +267,12 @@ class BaseTranslation(ABC):
         """
         return None
 
-    def get_structured_messages(self, data: dict) -> list["AllMessageValues"] | None:
+    def get_structured_messages(self, data: dict, *, keep_tool_errors: bool = False) -> list["AllMessageValues"] | None:
         """
         Convert request data to OpenAI-spec structured messages.
 
-        Override in subclasses for format-specific conversion.
+        Override in subclasses for format-specific conversion. ``keep_tool_errors`` copies a provider's
+        tool-failure marker onto tool messages as ``is_error``; only for copies that are never sent to a model.
 
         Returns None if no convertible content is found.
         """

litellm/llms/base_llm/managed_resources/isolation.py

@@ -11,8 +11,9 @@ are denied so an empty user_id can never select an unscoped query.
 """
 
 from collections.abc import Mapping, Sequence
-from typing import Any, Final
+from typing import Final, Protocol
 
+from litellm.litellm_core_utils.core_helpers import require_attributes
 from litellm.types.proxy.auth.user_api_key_auth import (
     UserAPIKeyAuth,
 )
@@ -47,15 +48,20 @@ def resolve_resource_owner_id(
     return None
 
 
-def build_list_page(items: list[Any], has_more: bool = False) -> dict[str, object]:
+class _Identified(Protocol):
+    @property
+    def id(self) -> object: ...
+
+
+def build_list_page(items: Sequence[object], has_more: bool = False) -> dict[str, object]:
     """Build the OpenAI-style paginated list response shape used by managed
     file/batch/vector-store listings. ``first_id`` and ``last_id`` are
     sourced from each item's ``.id`` attribute."""
     return {
         "object": "list",
         "data": items,
-        "first_id": items[0].id if items else None,
-        "last_id": items[-1].id if items else None,
+        "first_id": require_attributes(items[0], _Identified, "id").id if items else None,
+        "last_id": require_attributes(items[-1], _Identified, "id").id if items else None,
         "has_more": has_more,
     }
 

litellm/llms/base_llm/responses/codex_compat.py

@@ -10,6 +10,11 @@ variant``. Both Amazon Bedrock endpoints reject them:
 They are *history* items, so they only appear from the second turn of a session
 onward -- a first-turn request succeeds and hides the problem entirely.
 
+An ``agent_message`` content block's ``encrypted_content`` is ciphertext that only the
+backend that issued it can decrypt for the recipient model. A backend that declares
+``supports_encrypted_agent_messages`` gets those items intact, since turning them into
+text would hand the model ciphertext it believes it wrote.
+
 Codex also sends a ``web_search`` tool on every turn. api.openai.com runs that tool
 itself; a backend with no server-side tools rejects the whole request over it, so
 the same providers drop the tool types their backend does not accept.
@@ -24,6 +29,7 @@ from typing import Final
 
 from typing_extensions import ReadOnly, TypedDict
 
+from litellm.litellm_core_utils.core_helpers import as_object_list, as_str_mapping
 from litellm.types.llms.openai import ResponseInputParam
 
 AGENT_MESSAGE_INPUT_ITEM_TYPE: Final = "agent_message"
@@ -63,6 +69,15 @@ def _agent_message_text(item: "Mapping[str, object]") -> str:
     )
 
 
+def _is_encrypted_block(block: object) -> bool:
+    mapping: Final = as_str_mapping(block)
+    return mapping is not None and bool(mapping.get("encrypted_content"))
+
+
+def _carries_encrypted_content(item: "Mapping[str, object]") -> bool:
+    return any(_is_encrypted_block(block) for block in as_object_list(item.get("content")) or ())
+
+
 def _normalize_agent_message_item(item: "Mapping[str, object]") -> "_RewrittenAssistantMessageItem | None":
     text: Final = _agent_message_text(item)
     if not text:
@@ -97,28 +112,35 @@ def _normalize_local_shell_call_item(item: "Mapping[str, object]") -> "_Rewritte
     return rewritten
 
 
-def _normalize_input_item(item: object) -> "tuple[object, str | None]":
+def _normalize_input_item(item: object, keep_encrypted_agent_messages: bool) -> "tuple[object, str | None]":
     """Returns (normalized item, or None to drop it; original type when rewritten)."""
-    if not isinstance(item, dict):
+    mapping: Final = as_str_mapping(item)
+    if mapping is None:
         return item, None
-    item_type: Final = item.get("type")
+    item_type: Final = mapping.get("type")
     if item_type == AGENT_MESSAGE_INPUT_ITEM_TYPE:
-        return _normalize_agent_message_item(item), item_type
+        if keep_encrypted_agent_messages and _carries_encrypted_content(mapping):
+            return item, None
+        return _normalize_agent_message_item(mapping), AGENT_MESSAGE_INPUT_ITEM_TYPE
     if item_type == CONTEXT_COMPACTION_INPUT_ITEM_TYPE:
-        return _normalize_context_compaction_item(item), item_type
+        return _normalize_context_compaction_item(mapping), CONTEXT_COMPACTION_INPUT_ITEM_TYPE
     if item_type == LOCAL_SHELL_CALL_INPUT_ITEM_TYPE:
-        return _normalize_local_shell_call_item(item), item_type
+        return _normalize_local_shell_call_item(mapping), LOCAL_SHELL_CALL_INPUT_ITEM_TYPE
     return item, None
 
 
 def normalize_codex_input_items(
     input: "str | ResponseInputParam",
+    *,
+    keep_encrypted_agent_messages: bool,
 ) -> "tuple[str | ResponseInputParam, tuple[str, ...]]":
     """Rewrite the Codex history item types a Responses backend rejects.
 
-    ``agent_message`` (Codex multi-agent traffic; its ``encrypted_content`` slot
-    carries the plaintext payload when the model never issued encrypted args)
-    becomes an assistant message, ``context_compaction`` becomes the ``compaction``
+    ``agent_message`` (Codex multi-agent traffic) carrying an ``encrypted_content``
+    block is forwarded intact when ``keep_encrypted_agent_messages`` is set, because
+    only the backend can read it. Otherwise it becomes an assistant message: a backend
+    that never encrypts gets the plaintext payload Codex puts in that slot when the
+    model issued no encrypted args. ``context_compaction`` becomes the ``compaction``
     spelling these backends accept, and ``local_shell_call`` becomes the
     ``function_call`` its recorded ``function_call_output`` already pairs with.
 
@@ -127,7 +149,7 @@ def normalize_codex_input_items(
     """
     if not isinstance(input, list):
         return input, ()
-    normalized: Final = tuple(_normalize_input_item(item) for item in input)
+    normalized: Final = tuple(_normalize_input_item(item, keep_encrypted_agent_messages) for item in input)
     rewritten_types: Final = tuple(sorted(frozenset(item_type for _, item_type in normalized if item_type is not None)))
     kept: Final = [i for i, _ in normalized if i is not None]
     # Codex passthrough items sit outside the OpenAI input union.

litellm/llms/base_llm/videos/async_task_utils.py

@@ -7,11 +7,104 @@ from typing import Final
 
 from pydantic import TypeAdapter, ValidationError
 
+from litellm.exceptions import UnsupportedParamsError
 from litellm.images.utils import ImageEditRequestUtils
+from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoFrameImage, VideoInputReference
 from litellm.types.videos.utils import decode_video_id_with_provider, id_safe_name
 
 _FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object])
 _FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
+_FRAME_IMAGES_ADAPTER: Final = TypeAdapter(tuple[VideoFrameImage, ...])
+_INPUT_REFERENCES_ADAPTER: Final = TypeAdapter(tuple[VideoInputReference, ...])
+OPENROUTER_VIDEO_FIELDS: Final = frozenset(
+    {"duration", "aspect_ratio", "frame_images", "input_references", "generate_audio", "seed"}
+)
+
+
+def unsupported_video_fields(
+    params: VideoCreateOptionalRequestParams,
+    model: str,
+    provider: str,
+    drop_params: bool,
+    supported_native: frozenset[str] = frozenset(),
+) -> frozenset[str]:
+    unsupported: Final = tuple(
+        key for key, value in params.items() if key in OPENROUTER_VIDEO_FIELDS - supported_native and value is not None
+    )
+    if unsupported and not drop_params:
+        raise UnsupportedParamsError(
+            message=f"{provider} video generation does not support {', '.join(unsupported)}, set drop_params=True to ignore them",
+            model=model,
+            llm_provider=provider,
+        )
+    return frozenset(unsupported)
+
+
+def video_image_params(
+    params: Mapping[str, object], model: str, provider: str, drop_params: bool
+) -> Mapping[str, object]:
+    frames: Final = params.get("frame_images")
+    references: Final = params.get("input_references")
+    conflict: Final = (
+        "content cannot be combined with frame_images or input_references"
+        if params.get("content") is not None and (frames is not None or references is not None)
+        else "frame_images cannot be combined with input_references or input_reference"
+        if frames is not None and (references is not None or params.get("input_reference") is not None)
+        else "input_references cannot be combined with input_reference"
+        if references is not None and params.get("input_reference") is not None
+        else None
+    )
+    if conflict is not None and not drop_params:
+        raise UnsupportedParamsError(
+            message=f"{conflict}, set drop_params=True to ignore it", model=model, llm_provider=provider
+        )
+    try:
+        parsed_frames: Final = _FRAME_IMAGES_ADAPTER.validate_python(() if frames is None else frames)
+        parsed_references: Final = _INPUT_REFERENCES_ADAPTER.validate_python(
+            () if references is None or frames is not None else references
+        )
+    except ValidationError as e:
+        if not drop_params:
+            raise UnsupportedParamsError(
+                message=f"Invalid video image items, set drop_params=True to ignore them. {e}",
+                model=model,
+                llm_provider=provider,
+            ) from e
+        return _mapped_video_image_params(
+            {k: v for k, v in params.items() if k not in {"frame_images", "input_references"}},
+            (),
+        )
+    roles: Final = tuple(item["frame_type"] for item in parsed_frames)
+    first_indices: Final = {role: roles.index(role) for role in frozenset(roles)}
+    if len(roles) != len(frozenset(roles)) and not drop_params:
+        raise UnsupportedParamsError(
+            message="Duplicate frame_type, set drop_params=True to ignore it", model=model, llm_provider=provider
+        )
+    frame_items: Final = tuple(
+        {"type": item["type"], "image_url": item["image_url"], "role": item["frame_type"]}
+        for index, item in enumerate(parsed_frames)
+        if index == first_indices[item["frame_type"]]
+    )
+    reference_items: Final = tuple(
+        {"type": item["type"], "image_url": item["image_url"], "role": "reference_image"} for item in parsed_references
+    )
+    media: Final = frame_items if frames is not None else reference_items
+    return _mapped_video_image_params(params, media)
+
+
+def _mapped_video_image_params(
+    params: Mapping[str, object], media: tuple[Mapping[str, object], ...]
+) -> Mapping[str, object]:
+    frames: Final = params.get("frame_images")
+    references: Final = params.get("input_references")
+    excluded: Final[frozenset[str]] = frozenset({"frame_images", "input_references", "aspect_ratio"}) | (
+        frozenset({"content", "input_reference"}) if frames is not None or references is not None else frozenset[str]()
+    )
+    return {
+        **{key: value for key, value in params.items() if key not in excluded},
+        **({"ratio": params["aspect_ratio"]} if params.get("aspect_ratio") is not None else {}),
+        **({"content": list(media)} if frames is not None or references is not None else {}),
+    }
 
 
 def _fields(value: object) -> Mapping[str, object] | None:

litellm/llms/bedrock/chat/converse_transformation.py

@@ -960,7 +960,8 @@ class AmazonConverseConfig(BaseConfig):
     ) -> bool:
         """Check if the Bedrock model takes native structured outputs (outputConfig.textFormat) for this request.
 
-        ``supports_native_structured_output`` in ``litellm.model_cost`` turns it on for every request, and
+        ``supports_native_structured_output`` or ``supports_bedrock_native_structured_output_by_default`` in
+        ``litellm.model_cost`` turns it on for every request, and
         ``supports_bedrock_native_structured_output`` (the deployment's ``model_info`` first) also when Claude
         will think, which the json tool path would drop. When the request does not ask for thinking, a model
         that cannot force the json tool gets it unless that key is false.

litellm/llms/bedrock/chat/invoke_handler.py

@@ -11,7 +11,7 @@ from litellm import verbose_logger
 from litellm._uuid import uuid
 from litellm.caching.caching import InMemoryCache
 from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
-from litellm.litellm_core_utils.core_helpers import map_finish_reason
+from litellm.litellm_core_utils.core_helpers import decoded_json_object, map_finish_reason
 from litellm.litellm_core_utils.litellm_logging import Logging
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import object_mapping
 from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers
@@ -50,6 +50,7 @@ from litellm.types.utils import GenericStreamingChunk as GChunk
 from ..common_utils import (
     BedrockError,
     BedrockEventStreamResponseDict,
+    bedrock_event_chunk_text,
     bedrock_event_stream_header,
     bedrock_event_stream_response,
     bedrock_stream_event_error_status,
@@ -59,6 +60,7 @@ from ..common_utils import (
     get_bedrock_response_stream_shape,
     get_bedrock_stream_event_statuses,
     get_bedrock_tool_name,
+    parse_bedrock_event_stream,
 )
 
 bedrock_tool_name_mappings: Final[InMemoryCache] = InMemoryCache(max_size_in_memory=50, default_ttl=600)
@@ -922,7 +924,7 @@ class AWSEventStreamDecoder:
         response_stream_shape: "Shape",
         request_id_headers: httpx.Headers | None = None,
     ) -> str | None:
-        parsed_response: Final = self.parser.parse(response_dict, response_stream_shape)
+        parsed_response: Final = parse_bedrock_event_stream(self.parser, response_dict, response_stream_shape)
 
         if response_dict["status_code"] != 200:
             raise build_bedrock_stream_error(response_dict, response_stream_shape, headers=request_id_headers)
@@ -932,11 +934,12 @@ class AWSEventStreamDecoder:
             raise build_bedrock_stream_event_error(
                 event_type, event_error_status, response_dict["body"], headers=request_id_headers
             )
-        if "chunk" in parsed_response:
-            chunk = parsed_response.get("chunk")
-            if not chunk:
+        parsed_fields: Final = decoded_json_object(parsed_response)
+        if "chunk" in parsed_fields:
+            event_chunk: Final = parsed_fields.get("chunk")
+            if not event_chunk:
                 return None
-            return chunk.get("bytes").decode()
+            return bedrock_event_chunk_text(event_chunk)
         else:
             chunk = response_dict.get("body")
             if not chunk:

litellm/llms/bedrock/chat/native_output_fallback.py

@@ -9,7 +9,9 @@ from litellm.llms.bedrock.common_utils import (
     NativeOutputFallback,
     bedrock_strict_tools_fallback,
     is_bedrock_native_output_rejection,
+    is_bedrock_stream_permission_denial,
     warn_bedrock_native_output_rejected,
+    warn_bedrock_native_output_stream_denied,
     warn_bedrock_strict_tools_dropped,
 )
 from litellm.types.utils import ModelResponse
@@ -25,22 +27,55 @@ NativeOutputFallbackFactory: TypeAlias = Callable[[], NativeOutputFallback | Non
 class _SelectedFallback:
     fallback: NativeOutputFallback
     native_output_retry: bool
+    buffered_native_output: bool = False
 
 
 _RetryFor: TypeAlias = Callable[[BaseLLMException], _SelectedFallback | None]
+_NextRetryFor: TypeAlias = Callable[[BaseLLMException, _SelectedFallback], _SelectedFallback | None]
 
 
-def _native_output_retry(
-    error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
+def _native_output_rejection_retry(
+    error: BaseLLMException, fallback: NativeOutputFallbackFactory, model: str
 ) -> NativeOutputFallback | None:
-    if fallback is None or not is_bedrock_native_output_rejection(error.status_code, error.message):
-        return None
     retry: Final = fallback()
     if retry is not None:
         warn_bedrock_native_output_rejected(model, error.message)
     return retry
 
 
+def _buffered(retry: NativeOutputFallback) -> NativeOutputFallback:
+    return NativeOutputFallback(
+        optional_params={**retry.optional_params, "fake_stream": True}, litellm_params=retry.litellm_params
+    )
+
+
+def _buffered_stream_retry(fallback: NativeOutputFallbackFactory, model: str) -> NativeOutputFallback | None:
+    retry: Final = fallback()
+    if retry is None or retry.optional_params.get("fake_stream") is not True:
+        return None
+    warn_bedrock_native_output_stream_denied(model)
+    return retry
+
+
+def _native_output_retry(
+    error: BaseLLMException,
+    fallback: NativeOutputFallbackFactory | None,
+    buffered_stream_fallback: NativeOutputFallbackFactory | None,
+    model: str,
+) -> _SelectedFallback | None:
+    if fallback is not None and is_bedrock_native_output_rejection(error.status_code, error.message):
+        rejection_retry: Final = _native_output_rejection_retry(error, fallback, model)
+        return None if rejection_retry is None else _SelectedFallback(rejection_retry, native_output_retry=True)
+    if buffered_stream_fallback is not None and is_bedrock_stream_permission_denial(error.status_code, error.message):
+        stream_retry: Final = _buffered_stream_retry(buffered_stream_fallback, model)
+        return (
+            None
+            if stream_retry is None
+            else _SelectedFallback(stream_retry, native_output_retry=True, buffered_native_output=True)
+        )
+    return None
+
+
 def _strict_tools_retry(
     error: BaseLLMException, fallback: NativeOutputFallbackFactory | None, model: str
 ) -> NativeOutputFallback | None:
@@ -55,12 +90,13 @@ def _strict_tools_retry(
 def _fallback_for(
     error: BaseLLMException,
     fallback: NativeOutputFallbackFactory | None,
+    buffered_stream_fallback: NativeOutputFallbackFactory | None,
     strict_tools_fallback: NativeOutputFallbackFactory | None,
     model: str,
 ) -> _SelectedFallback | None:
-    native_output_retry: Final = _native_output_retry(error, fallback, model)
+    native_output_retry: Final = _native_output_retry(error, fallback, buffered_stream_fallback, model)
     if native_output_retry is not None:
-        return _SelectedFallback(native_output_retry, native_output_retry=True)
+        return native_output_retry
     strict_tools_retry: Final = _strict_tools_retry(error, strict_tools_fallback, model)
     if strict_tools_retry is None:
         return None
@@ -87,12 +123,81 @@ def _strict_tools_retry_after_native_output_retry(
     return composed_retry
 
 
+def _next_retry(
+    error: BaseLLMException,
+    retry: _SelectedFallback,
+    fallback: NativeOutputFallbackFactory | None,
+    strict_tools_fallback: NativeOutputFallbackFactory | None,
+    model: str,
+) -> _SelectedFallback | None:
+    if (
+        retry.buffered_native_output
+        and fallback is not None
+        and is_bedrock_native_output_rejection(error.status_code, error.message)
+    ):
+        rejection_retry: Final = _native_output_rejection_retry(error, fallback, model)
+        return (
+            None if rejection_retry is None else _SelectedFallback(_buffered(rejection_retry), native_output_retry=True)
+        )
+    composed_retry: Final = _strict_tools_retry_after_native_output_retry(error, retry, strict_tools_fallback, model)
+    return None if composed_retry is None else _SelectedFallback(composed_retry, native_output_retry=False)
+
+
+def _attempt_or_next_retry(
+    attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor
+) -> NativeOutputAttemptResult | _SelectedFallback:
+    try:
+        return attempt(retry.fallback)
+    except BaseLLMException as error:
+        next_retry: Final = next_retry_for(error, retry)
+        if next_retry is None:
+            raise
+        return next_retry
+
+
+def _attempt_retries(
+    attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor
+) -> NativeOutputAttemptResult:
+    after_first_retry: Final = _attempt_or_next_retry(attempt, retry, next_retry_for)
+    if not isinstance(after_first_retry, _SelectedFallback):
+        return after_first_retry
+    after_second_retry: Final = _attempt_or_next_retry(attempt, after_first_retry, next_retry_for)
+    if not isinstance(after_second_retry, _SelectedFallback):
+        return after_second_retry
+    return attempt(after_second_retry.fallback)
+
+
+async def _await_or_next_retry(
+    attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor
+) -> NativeOutputResponse | _SelectedFallback:
+    try:
+        result: Final = attempt(retry.fallback)
+        return await result if isinstance(result, Coroutine) else result
+    except BaseLLMException as error:
+        next_retry: Final = next_retry_for(error, retry)
+        if next_retry is None:
+            raise
+        return next_retry
+
+
+async def _await_retries(
+    attempt: NativeOutputAttempt, retry: _SelectedFallback, next_retry_for: _NextRetryFor
+) -> NativeOutputResponse:
+    after_first_retry: Final = await _await_or_next_retry(attempt, retry, next_retry_for)
+    if not isinstance(after_first_retry, _SelectedFallback):
+        return after_first_retry
+    after_second_retry: Final = await _await_or_next_retry(attempt, after_first_retry, next_retry_for)
+    if not isinstance(after_second_retry, _SelectedFallback):
+        return after_second_retry
+    last_attempt: Final = attempt(after_second_retry.fallback)
+    return await last_attempt if isinstance(last_attempt, Coroutine) else last_attempt
+
+
 async def _await_with_native_output_fallback(
     first_attempt: Coroutine[object, object, NativeOutputResponse],
     attempt: NativeOutputAttempt,
     retry_for: _RetryFor,
-    strict_tools_fallback: NativeOutputFallbackFactory | None,
-    model: str,
+    next_retry_for: _NextRetryFor,
 ) -> NativeOutputResponse:
     try:
         return await first_attempt
@@ -100,17 +205,7 @@ async def _await_with_native_output_fallback(
         retry: Final = retry_for(error)
         if retry is None:
             raise
-    try:
-        second_attempt: Final = attempt(retry.fallback)
-        return await second_attempt if isinstance(second_attempt, Coroutine) else second_attempt
-    except BaseLLMException as error:
-        composed_retry: Final = _strict_tools_retry_after_native_output_retry(
-            error, retry, strict_tools_fallback, model
-        )
-        if composed_retry is None:
-            raise
-    third_attempt: Final = attempt(composed_retry)
-    return await third_attempt if isinstance(third_attempt, Coroutine) else third_attempt
+    return await _await_retries(attempt, retry, next_retry_for)
 
 
 def call_with_native_output_fallback(
@@ -118,11 +213,19 @@ def call_with_native_output_fallback(
     fallback: NativeOutputFallbackFactory | None,
     model: str,
     strict_tools_fallback: NativeOutputFallbackFactory | None = None,
+    buffered_stream_fallback: NativeOutputFallbackFactory | None = None,
 ) -> NativeOutputAttemptResult:
-    if fallback is None and strict_tools_fallback is None:
+    if fallback is None and strict_tools_fallback is None and buffered_stream_fallback is None:
         return attempt(None)
     retry_for: Final = partial(
-        _fallback_for, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model
+        _fallback_for,
+        fallback=fallback,
+        buffered_stream_fallback=buffered_stream_fallback,
+        strict_tools_fallback=strict_tools_fallback,
+        model=model,
+    )
+    next_retry_for: Final = partial(
+        _next_retry, fallback=fallback, strict_tools_fallback=strict_tools_fallback, model=model
     )
     try:
         first_attempt: Final = attempt(None)
@@ -130,15 +233,7 @@ def call_with_native_output_fallback(
         retry: Final = retry_for(error)
         if retry is None:
             raise
-        try:
-            return attempt(retry.fallback)
-        except BaseLLMException as retry_error:
-            composed_retry: Final = _strict_tools_retry_after_native_output_retry(
-                retry_error, retry, strict_tools_fallback, model
-            )
-            if composed_retry is None:
-                raise
-            return attempt(composed_retry)
+        return _attempt_retries(attempt, retry, next_retry_for)
     if isinstance(first_attempt, Coroutine):
-        return _await_with_native_output_fallback(first_attempt, attempt, retry_for, strict_tools_fallback, model)
+        return _await_with_native_output_fallback(first_attempt, attempt, retry_for, next_retry_for)
     return first_attempt

litellm/llms/bedrock/common_utils.py

@@ -13,7 +13,7 @@ import warnings
 from collections.abc import Callable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias
+from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias
 
 from typing_extensions import (
     ReadOnly,
@@ -24,6 +24,7 @@ from typing_extensions import (
 if TYPE_CHECKING:
     from botocore.eventstream import EventStreamMessage
     from botocore.model import ServiceModel, Shape
+    from botocore.parsers import EventStreamJSONParser
 
     from litellm.types.llms.bedrock import BedrockCreateBatchRequest
 
@@ -33,7 +34,7 @@ from pydantic import ConfigDict, TypeAdapter, ValidationError
 import litellm
 from litellm import verbose_logger
 from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix
-from litellm.litellm_core_utils.core_helpers import lax_bool
+from litellm.litellm_core_utils.core_helpers import decoded_json_object, lax_bool, require_attributes
 from litellm.litellm_core_utils.optional_imports import ensure_optional_import
 from litellm.litellm_core_utils.prompt_templates.json_schema_utils import (
     schema_property_count,
@@ -135,7 +136,7 @@ _BEDROCK_AWS_AUTH_PARAMETER_KEYS: Final[tuple[str, ...]] = (*AWS_AUTH_PARAM_KEYS
 def merge_bedrock_aws_request_params(
     litellm_params: Mapping[str, object],
     optional_params: Mapping[str, object],
-) -> dict[str, Any]:
+) -> dict[str, object]:
     """Merge deployment and request parameters without allowing auth escalation.
 
     Deployment configuration is authoritative for AWS authentication. When a
@@ -297,9 +298,16 @@ def _bedrock_model_supports(model: str, key: str, custom_llm_provider: str = "be
 def _bedrock_native_output_enabled(
     model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None
 ) -> bool:
-    deployment_flag: Final = (model_info or {}).get("supports_native_structured_output")
+    deployment_info: Final = model_info or {}
+    deployment_flag: Final = deployment_info.get("supports_native_structured_output")
     if isinstance(deployment_flag, bool):
         return deployment_flag
+    return deployment_info.get("supports_bedrock_native_structured_output") is not False and (
+        bedrock_model_always_takes_native_output(model, custom_llm_provider)
+    )
+
+
+def bedrock_model_always_takes_native_output(model: str, custom_llm_provider: str = "bedrock") -> bool:
     return _bedrock_model_supports(model, "supports_native_structured_output", custom_llm_provider)
 
 
@@ -443,6 +451,18 @@ def warn_bedrock_native_output_rejected(model: str, error_text: str) -> None:
     )
 
 
+def is_bedrock_stream_permission_denial(status_code: int, error_text: str) -> bool:
+    return status_code == 403 and "bedrock:InvokeModelWithResponseStream" in error_text
+
+
+def warn_bedrock_native_output_stream_denied(model: str) -> None:
+    verbose_logger.warning(
+        "Bedrock denied bedrock:InvokeModelWithResponseStream for model=%s, so litellm sent the structured output "
+        "request again as a buffered stream. Allow that action to stream it.",
+        model,
+    )
+
+
 @dataclass(frozen=True, slots=True)
 class NativeOutputFallback:
     optional_params: Mapping[str, object]
@@ -543,6 +563,22 @@ def json_object_or_none(value: object) -> Mapping[str, object] | None:
 BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES: Final = 64
 
 
+def _response_format_json_schema(response_format: object) -> Mapping[str, object] | None:
+    json_schema: Final = json_object_or_none((json_object_or_none(response_format) or {}).get("json_schema"))
+    return json_object_or_none((json_schema or {}).get("schema"))
+
+
+def bedrock_batch_record_takes_native_output(
+    model: str, response_format: object, model_info: Mapping[str, object] | None = None
+) -> bool:
+    schema: Final = _response_format_json_schema(response_format)
+    return _bedrock_native_output_enabled(model, "bedrock", model_info) and (
+        schema is None
+        or schema_property_count(schema) <= BEDROCK_NATIVE_OUTPUT_MAX_PROPERTIES
+        or not _bedrock_model_supports(model, "supports_bedrock_native_structured_output_by_default")
+    )
+
+
 def bedrock_cleaned_output_schema(
     model: str, schema: Mapping[str, object], custom_llm_provider: str = "bedrock"
 ) -> Mapping[str, object] | None:
@@ -2481,15 +2517,16 @@ class BedrockEventStreamDecoderBase:
                 ),
             )
         response_dict: Final = event.to_response_dict()
-        parsed_response: Final = self.parser.parse(response_dict, response_stream_shape)
+        parsed_response: Final = parse_bedrock_event_stream(self.parser, response_dict, response_stream_shape)
 
         if response_dict["status_code"] != 200:
             raise build_bedrock_stream_error(response_dict, response_stream_shape)
-        if "chunk" in parsed_response:
-            chunk = parsed_response.get("chunk")
-            if not chunk:
+        parsed_fields: Final = decoded_json_object(parsed_response)
+        if "chunk" in parsed_fields:
+            event_chunk: Final = parsed_fields.get("chunk")
+            if not event_chunk:
                 return None
-            return chunk.get("bytes").decode()
+            return bedrock_event_chunk_text(event_chunk)
         else:
             chunk = response_dict.get("body")
             if not chunk:
@@ -2506,6 +2543,20 @@ def _decoded_json_value(raw: str) -> object:
     return _JSON_VALUE.validate_python(json.loads(raw))
 
 
+class _DecodablePayload(Protocol):
+    def decode(self) -> str: ...
+
+
+def parse_bedrock_event_stream(
+    parser: EventStreamJSONParser, response_dict: Mapping[str, object], response_stream_shape: Shape
+) -> object:
+    return _JSON_VALUE.validate_python(parser.parse(response_dict, response_stream_shape))
+
+
+def bedrock_event_chunk_text(chunk: object) -> str:
+    return require_attributes(decoded_json_object(chunk).get("bytes"), _DecodablePayload, "decode").decode()
+
+
 def get_anthropic_beta_from_headers(headers: dict) -> list[str]:
     """
     Extract anthropic-beta header values and convert them to a list.
@@ -2785,3 +2836,13 @@ class CommonBatchFilesUtils:
         Get Bedrock-specific error class.
         """
         return BedrockError(status_code=status_code, message=error_message, headers=headers)
+
+
+def bedrock_model_encrypts_agent_messages(model: str) -> bool:
+    """Only OpenAI's GPT models on Bedrock lock Codex worker messages.
+
+    Open-weight models (gpt-oss, Gemma, Grok) never lock them, so Codex puts plain text in the locked
+    slot, which Bedrock rejects with ``invalid_encrypted_content`` unless it is rewritten to text first.
+    """
+    name: Final = f".{model.rsplit('/', 1)[-1]}"
+    return ".openai.gpt-" in name and ".openai.gpt-oss" not in name

litellm/llms/bedrock/count_tokens/bedrock_token_counter.py

@@ -45,7 +45,7 @@ class _CountTokensReply(LiteLLMPydanticObjectBase):
 @dataclass(frozen=True, slots=True)
 class CountedTokens:
     input_tokens: int
-    original_response: Mapping[str, JsonValue]
+    original_response: Mapping[str, object]
     tokenizer_type: str
 
 

litellm/llms/bedrock/count_tokens/handler.py

@@ -7,14 +7,18 @@ Simplified handler leveraging existing LiteLLM Bedrock infrastructure.
 from typing import Any, Final
 
 import httpx
+from pydantic import TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.core_helpers import decoded_json_object
 from litellm.llms.bedrock.base_aws_llm import run_aws_signing
 from litellm.llms.bedrock.common_utils import BedrockError
 from litellm.llms.bedrock.count_tokens.transformation import BedrockCountTokensConfig
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
 
+_DECODED_JSON: Final = TypeAdapter(object)
+
 
 class BedrockCountTokensHandler(BedrockCountTokensConfig):
     """
@@ -29,7 +33,7 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
         litellm_params: dict[str, Any],
         resolved_model: str,
         client: AsyncHTTPHandler | None = None,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Handle a CountTokens request using existing LiteLLM patterns.
 
@@ -111,12 +115,12 @@ class BedrockCountTokensHandler(BedrockCountTokensConfig):
                     response=response,
                 )
 
-            bedrock_response: Final = response.json()
+            bedrock_response: Final = _DECODED_JSON.validate_python(response.json())
 
             verbose_logger.debug("Bedrock response: %s", bedrock_response)
 
             # Transform response back to expected format
-            final_response: Final = self.transform_bedrock_response_to_anthropic(bedrock_response)
+            final_response: Final = self.transform_bedrock_response_to_anthropic(decoded_json_object(bedrock_response))
 
             verbose_logger.debug("Final response: %s", final_response)
 

litellm/llms/bedrock/count_tokens/transformation.py

@@ -247,9 +247,7 @@ class BedrockCountTokensConfig(BaseAWSLLM):
         )
         return join_api_base(base_url, f"model/{encoded_model_id}/count-tokens")
 
-    def transform_bedrock_response_to_anthropic(
-        self, bedrock_response: Mapping[str, JsonValue]
-    ) -> dict[str, JsonValue]:
+    def transform_bedrock_response_to_anthropic(self, bedrock_response: Mapping[str, object]) -> dict[str, object]:
         """
         Transform Bedrock CountTokens response to Anthropic format.
 

litellm/llms/bedrock/files/transformation.py

@@ -11,7 +11,7 @@ from datetime import datetime
 from email.utils import parsedate_to_datetime
 from itertools import chain
 from types import MappingProxyType
-from typing import Any, Final, Literal, TypeAlias, TypedDict
+from typing import Any, Final, Literal, Protocol, TypeAlias, TypedDict
 from urllib.parse import quote, unquote, urlencode
 
 import httpx
@@ -19,6 +19,7 @@ from httpx import Headers, Response
 from openai.types.file_deleted import FileDeleted
 from pydantic import ConfigDict, Field, TypeAdapter
 from typing_extensions import ReadOnly
+from typing_extensions import TypedDict as ExtraItemsTypedDict
 
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
@@ -38,6 +39,7 @@ from litellm.litellm_core_utils.cloud_storage_security import (
     split_configured_cloud_bucket_name,
     validate_managed_cloud_file_id,
 )
+from litellm.litellm_core_utils.core_helpers import require_attributes
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     text_completion_prompt_to_messages,
 )
@@ -71,6 +73,7 @@ from litellm.utils import get_llm_provider, get_optional_params
 from ..base_aws_llm import BaseAWSLLM
 from ..common_utils import (
     BedrockError,
+    bedrock_batch_record_takes_native_output,
     deployment_model_info,
     merge_bedrock_aws_request_params,
     resolve_s3_encryption_key_id,
@@ -150,6 +153,10 @@ def _retrieved_s3_file_size(raw_response: Response) -> int:
     )
 
 
+class _SupportsRstrip(Protocol):
+    def rstrip(self, chars: str, /) -> str: ...
+
+
 class _S3DeleteContext(LiteLLMBaseModel):
     file_id: str = Field(min_length=1)
 
@@ -167,6 +174,24 @@ _LITELLM_PARAMS_THE_MAPPER_TAKES: Final = frozenset({"allowed_openai_params"})
 _MAPPED_PARAMS_THE_REQUEST_HANDLER_STRIPS: Final = frozenset({"json_mode"})
 
 
+class _ParamsTheMapperOwns(ExtraItemsTypedDict, total=False, extra_items=object):
+    pass
+
+
+_UPLOAD_PARAMS: Final = TypeAdapter(dict[str, object])
+
+
+def _upload_params(params: object) -> Mapping[str, object]:
+    return _UPLOAD_PARAMS.validate_python(params) if isinstance(params, Mapping) else {}
+
+
+def _upload_deployment_model_info(litellm_params: object) -> Mapping[str, object]:
+    upload_params: Final = _upload_params(litellm_params)
+    return deployment_model_info(upload_params) or deployment_model_info(
+        _upload_params(upload_params.get("litellm_metadata"))
+    )
+
+
 def _invoke_route_model(model: str) -> str:
     return f"invoke/{_strip_llm_routing_prefix(model).removeprefix('invoke/')}"
 
@@ -695,9 +720,11 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         encoded_object_name: Final = encode_s3_object_key_for_url(object_name)
 
         # S3 endpoint URL format
-        s3_endpoint_url: Final = (
+        s3_endpoint_url: Final = require_attributes(
             request_params.get("s3_endpoint_url")
-            or f"https://s3.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}"
+            or f"https://s3.{aws_region_name}.{get_aws_dns_suffix(aws_region_name)}",
+            _SupportsRstrip,
+            "rstrip",
         ).rstrip("/")
 
         return f"{s3_endpoint_url}/{bucket_name}/{encoded_object_name}"
@@ -1034,6 +1061,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         openai_request_body: Mapping[str, Any],
         model: str,
         provider: str | None = None,
+        model_info: Mapping[str, object] | None = None,
     ) -> dict[str, object]:
         """
         Transform OpenAI request body to Bedrock-compatible modelInput
@@ -1046,7 +1074,8 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         from litellm.types.utils import LlmProviders
 
         messages: Final = map_developer_role_to_system_role(openai_request_body.get("messages", []))
-        optional_params: Final = {k: v for k, v in openai_request_body.items() if k not in ["model", "messages"]}
+        request_params: Final[Mapping[str, object]] = openai_request_body
+        optional_params: Final = {k: v for k, v in request_params.items() if k not in ["model", "messages"]}
 
         # --- Anthropic: use existing AmazonAnthropicClaudeConfig ---
         if provider == LlmProviders.ANTHROPIC:
@@ -1055,14 +1084,22 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
             )
 
             config: Final = AmazonAnthropicClaudeConfig()
-            litellm_params: Final = without_bedrock_native_output_opt_in({})
+            litellm_params: Final = without_bedrock_native_output_opt_in(
+                {
+                    "model_info": {
+                        "supports_native_structured_output": bedrock_batch_record_takes_native_output(
+                            _invoke_route_model(model), optional_params.get("response_format"), model_info
+                        )
+                    }
+                }
+            )
             mapped_params = get_optional_params(
                 model=_invoke_route_model(model),
                 custom_llm_provider="bedrock",
                 messages=messages,
                 model_info=deployment_model_info(litellm_params),
-                **MappingProxyType(
-                    {
+                **_ParamsTheMapperOwns(
+                    **{
                         k: v
                         for k, v in optional_params.items()
                         if not is_litellm_owned_kwarg(k) or k in _LITELLM_PARAMS_THE_MAPPER_TAKES
@@ -1150,25 +1187,32 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         ]
 
     def transform_batch_input_lines(
-        self, batch_input_lines: Sequence[_BatchInputLine], target_model: str = ""
+        self,
+        batch_input_lines: Sequence[_BatchInputLine],
+        target_model: str = "",
+        model_info: Mapping[str, object] | None = None,
     ) -> tuple[_BedrockBatchRecord, ...]:
         return tuple(
-            self._transform_batch_input_line(idx, batch_input_line, target_model)
+            self._transform_batch_input_line(idx, batch_input_line, target_model, model_info)
             for idx, batch_input_line in enumerate(batch_input_lines)
         )
 
     def _transform_batch_input_line(
-        self, idx: int, batch_input_line: _BatchInputLine, target_model: str
+        self, idx: int, batch_input_line: _BatchInputLine, target_model: str, model_info: Mapping[str, object] | None
     ) -> _BedrockBatchRecord:
         try:
-            return self._transform_openai_batch_record(idx, batch_input_line.record, target_model)
+            return self._transform_openai_batch_record(idx, batch_input_line.record, target_model, model_info)
         except _BATCH_RECORD_CONVERSION_ERRORS as e:
             raise _batch_line_error(
                 batch_input_line.line_number, f"could not be converted to a Bedrock batch record: {e}"
             ) from e
 
     def _transform_openai_batch_record(
-        self, idx: int, openai_record: _OpenAIBatchRecord, target_model: str
+        self,
+        idx: int,
+        openai_record: _OpenAIBatchRecord,
+        target_model: str,
+        model_info: Mapping[str, object] | None = None,
     ) -> _BedrockBatchRecord:
         import litellm
 
@@ -1192,6 +1236,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
                 openai_request_body=self._transform_batch_body_to_chat_body(openai_body, record_kind),
                 model=model_for_transform,
                 provider=provider,
+                model_info=model_info,
             )
         )
         return _BedrockBatchRecord(
@@ -1211,7 +1256,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
         model: str,
         create_file_data: CreateFileRequest,
         optional_params: dict,
-        litellm_params: dict,
+        litellm_params: dict[str, object],
     ) -> bytes | str | dict:
         """
         Transform file request and return a pre-signed request for S3.
@@ -1237,7 +1282,7 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig):
             litellm_params_model: Final = litellm_params.get("model")
             target_model: Final = model or (litellm_params_model if isinstance(litellm_params_model, str) else "")
             bedrock_jsonl_content: Final = self.transform_batch_input_lines(
-                batch_input_lines, target_model=target_model
+                batch_input_lines, target_model=target_model, model_info=_upload_deployment_model_info(litellm_params)
             )
             file_content = "\n".join(json.dumps(item) for item in bedrock_jsonl_content)
         elif isinstance(extracted_file_data_content, bytes):