Merge branch 'BerriAI:main' into fix-redis-cache-default

.github/workflows/test-linting.yml

@@ -600,6 +600,9 @@ jobs:
       - name: check_unsafe_enterprise_import
         run: uv run --no-sync python ./tests/code_coverage_tests/check_unsafe_enterprise_import.py
 
+      - name: check_telemetry_imports
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_telemetry_imports.py
+
       - name: ban_copy_deepcopy_kwargs
         run: uv run --no-sync python ./tests/code_coverage_tests/ban_copy_deepcopy_kwargs.py
 
@@ -615,6 +618,9 @@ jobs:
       - name: check_migrations_no_data_rewrites
         run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_no_data_rewrites.py
 
+      - name: check_migrations_lock_timeout
+        run: uv run --no-sync python ./tests/code_coverage_tests/check_migrations_lock_timeout.py
+
       - name: check_no_publicly_known_master_key
         run: uv run --no-sync python ./tests/code_coverage_tests/check_no_publicly_known_master_key.py
 

.github/workflows/test-unit.yml

@@ -233,6 +233,7 @@ jobs:
               tests/unit/llms/base_llm
               tests/unit/llms/baseten
               tests/unit/llms/black_forest_labs
+              tests/unit/llms/brave
               tests/unit/llms/bytez
               tests/unit/llms/cerebras
               tests/unit/llms/chat
@@ -268,6 +269,7 @@ jobs:
               tests/unit/llms/gdc
               tests/unit/llms/gemini
               tests/unit/llms/gigachat
+              tests/unit/llms/google_pse
               tests/unit/llms/github_copilot
               tests/unit/llms/gradient_ai
               tests/unit/llms/groq
@@ -300,6 +302,7 @@ jobs:
               tests/unit/llms/nebius
               tests/unit/llms/neosantara
               tests/unit/llms/nimble
+              tests/unit/llms/nlp_cloud
               tests/unit/llms/novita
               tests/unit/llms/nscale
               tests/unit/llms/nvidia_nim
@@ -310,6 +313,7 @@ jobs:
               tests/unit/llms/oobabooga
               tests/unit/llms/openai_like
               tests/unit/llms/opencode
+              tests/unit/llms/opensandbox
               tests/unit/llms/openrouter
               tests/unit/llms/ovhcloud
               tests/unit/llms/parallel_ai
@@ -446,6 +450,7 @@ jobs:
               tests/unit/repositories
               tests/unit/secret_managers
               tests/unit/skills/test_skills_main.py
+              tests/unit/telemetry tests/unit/proxy/telemetry
               tests/unit/tracing
             python-version: "3.12"
             workers: 2

AGENTS.md

@@ -23,7 +23,7 @@ When adding new features, add meaningful tests. Don't add tests that don't check
 
 Same thing for bug fixes. The tests should make it so that this specific bug can never happen again without failing tests (i.e., regression)
 
-Never test structure of code only function of it
+Never test structure or implementation details of code, only function of it. A test should catch a change that breaks behavior by accident. If the only way it can fail is someone deliberately undoing the change, don't write it
 
 A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent". Assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken
 

backend/routes/allowlist.py

@@ -32,6 +32,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = (
     "/token",
     "/onboarding/",
     "/audit",
+    "/telemetry/",
     "/oauth/",
     "/invitation/",
     "/jwt/",

ci_cd/run_migration.py

@@ -322,7 +322,12 @@ def create_migration(
 
                     # Write the SQL to migration.sql
                     migration_file = migration_dir / "migration.sql"
-                    migration_file.write_text(diff_sql)
+                    migration_sql: Final = (
+                        diff_sql
+                        if "CONCURRENTLY" in diff_sql.upper()
+                        else "SET LOCAL lock_timeout = '10s';\n\n" + diff_sql
+                    )
+                    migration_file.write_text(migration_sql)
 
                     print(f"Created migration in {migration_dir}")
                     return True

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.76"
+version = "0.1.77"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.76"
+version = "0.1.77"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

gateway/routes/allowlist.py

@@ -121,6 +121,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     # Realtime / streaming
     "/v1/realtime",
     "/realtime",
+    "/v1/live/sessions",
+    "/live/sessions",
     # Health & ops
     "/health",
     "/metrics",

helm/litellm-helm/values.yaml

@@ -440,10 +440,10 @@ db:
 # Prefer the native /health/drain preStop hook over a fixed `sleep`: it marks
 # the pod NotReady and blocks only until in-flight requests actually finish
 # (bounded by GRACEFUL_SHUTDOWN_TIMEOUT, default 30s), instead of always
-# waiting the worst-case duration. The drain runs once (the preStop hook and
-# the SIGTERM handler share it), so set terminationGracePeriodSeconds a few
-# seconds above GRACEFUL_SHUTDOWN_TIMEOUT to leave room for teardown before
-# SIGKILL.
+# waiting the worst-case duration. After SIGTERM the server waits up to
+# GRACEFUL_SHUTDOWN_TIMEOUT again for connections still open and then cancels
+# them, so set terminationGracePeriodSeconds a few seconds above twice
+# GRACEFUL_SHUTDOWN_TIMEOUT to leave room for teardown before SIGKILL.
 #
 # /health/drain is off by default; enable it with
 # general_settings.enable_drain_endpoint: true. The kubelet calls preStop

litellm-proxy-extras/litellm_proxy_extras/migration_lock.py

@@ -1,5 +1,4 @@
 import random
-import re
 import time
 from collections.abc import Generator, Mapping
 from contextlib import contextmanager
@@ -10,12 +9,10 @@ from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
 from litellm_proxy_extras._logging import logger
 from litellm_proxy_extras.prisma_toolchain import (
     MIGRATION_LOCK_TIMEOUT_ENV_VAR,
-    migration_ddl_lock_timeout,
     migration_lock_timeout,
 )
 
 MIGRATION_LOCK_KEY: Final = int.from_bytes(b"llm_mig2", "big")
-_LOCK_TIMEOUT_PINNED_RE: Final = re.compile(r"(?:-c\s*|--)lock_timeout=")
 
 if TYPE_CHECKING:
     import psycopg
@@ -30,29 +27,12 @@ def _migration_url(database_url: str, direct_url: "str | None") -> str:
     return urlunsplit(direct._replace(query=urlencode((*parameters, ("schema", schema)))))
 
 
-def _with_ddl_lock_timeout(url: str) -> str:
-    split: Final = urlsplit(url)
-    pairs: Final = tuple(parse_qsl(split.query))
-    if any(key == "pgbouncer" and value == "true" for key, value in pairs):
-        return url
-    existing: Final = next((value for key, value in pairs if key == "options"), "")
-    if _LOCK_TIMEOUT_PINNED_RE.search(existing):
-        return url
-    timeout_ms: Final = max(1, int(migration_ddl_lock_timeout() * 1000))
-    options: Final = f"{existing} -c lock_timeout={timeout_ms}".strip()
-    rewritten: Final = tuple((key, value) for key, value in pairs if key != "options") + (("options", options),)
-    return urlunsplit(split._replace(query=urlencode(rewritten)))
-
-
 def migration_environment(environment: Mapping[str, str]) -> Mapping[str, str]:
     database_url: Final = environment.get("DATABASE_URL")
     if not database_url:
         return environment
     migration_url: Final = _migration_url(database_url, environment.get("DIRECT_URL"))
-    return {
-        **environment,
-        "DATABASE_URL": _with_ddl_lock_timeout(migration_url),
-    }
+    return {**environment, "DATABASE_URL": migration_url}
 
 
 @dataclass(frozen=True, slots=True)

litellm-proxy-extras/litellm_proxy_extras/migrations/20261009000100_telemetry_reports/migration.sql

@@ -0,0 +1,12 @@
+CREATE TABLE IF NOT EXISTS "LiteLLM_TelemetryReport" (
+    "id" TEXT NOT NULL,
+    "window_start" DOUBLE PRECISION NOT NULL,
+    "window_end" DOUBLE PRECISION NOT NULL,
+    "report" JSONB NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    CONSTRAINT "LiteLLM_TelemetryReport_pkey" PRIMARY KEY ("id")
+);
+
+CREATE INDEX IF NOT EXISTS "LiteLLM_TelemetryReport_window_end_idx" ON "LiteLLM_TelemetryReport"("window_end");
+
+CREATE INDEX IF NOT EXISTS "LiteLLM_TelemetryReport_created_at_idx" ON "LiteLLM_TelemetryReport"("created_at");

litellm-proxy-extras/litellm_proxy_extras/prisma_toolchain.py

@@ -60,13 +60,11 @@ PRISMA_COMMAND_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_COMMAND_TIMEOUT"
 PRISMA_BOOTSTRAP_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_BOOTSTRAP_TIMEOUT"
 PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR = "LITELLM_PRISMA_MIGRATE_DEPLOY_TIMEOUT"
 MIGRATION_LOCK_TIMEOUT_ENV_VAR = "LITELLM_MIGRATION_LOCK_TIMEOUT"
-MIGRATION_DDL_LOCK_TIMEOUT_ENV_VAR = "LITELLM_MIGRATION_DDL_LOCK_TIMEOUT"
 NODEENV_CACHE_DIR_ENV_VAR = "PRISMA_NODEENV_CACHE_DIR"
 
 DEFAULT_PRISMA_COMMAND_TIMEOUT = 60.0
 DEFAULT_PRISMA_BOOTSTRAP_TIMEOUT = 600.0
 DEFAULT_PRISMA_MIGRATE_DEPLOY_TIMEOUT = 600.0
-DEFAULT_MIGRATION_DDL_LOCK_TIMEOUT = 10.0
 
 BOOTSTRAP_ARG = "--version"
 PRISMA_CONSOLE_SCRIPT = "prisma"
@@ -113,11 +111,6 @@ def migration_lock_timeout() -> float:
     return _timeout_from_env(MIGRATION_LOCK_TIMEOUT_ENV_VAR, 600.0)
 
 
-def migration_ddl_lock_timeout() -> float:
-    """Seconds one migration statement may wait on a table lock before Postgres cancels it."""
-    return _timeout_from_env(MIGRATION_DDL_LOCK_TIMEOUT_ENV_VAR, DEFAULT_MIGRATION_DDL_LOCK_TIMEOUT)
-
-
 def prisma_bootstrap_timeout() -> float:
     """Seconds the one-time Node toolchain install may run for."""
     return _timeout_from_env(

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -2045,3 +2045,14 @@ model LiteLLM_LensTraceSignal {
 
   @@id([trace_id, trace_ref])
 }
+
+model LiteLLM_TelemetryReport {
+  id           String   @id
+  window_start Float
+  window_end   Float
+  report       Json
+  created_at   DateTime @default(now())
+
+  @@index([window_end])
+  @@index([created_at])
+}

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -18,7 +18,6 @@ from litellm_proxy_extras import prisma_toolchain
 from litellm_proxy_extras._logging import logger
 from litellm_proxy_extras.migration_lock import held_migration_lock
 from litellm_proxy_extras.prisma_toolchain import (
-    MIGRATION_DDL_LOCK_TIMEOUT_ENV_VAR,
     PRISMA_COMMAND_TIMEOUT_ENV_VAR,
     PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR,
     ensure_prisma_toolchain,
@@ -1165,10 +1164,9 @@ class ProxyExtrasDBManager:
             raise RuntimeError(
                 f"Database migration failed after {MAX_MIGRATE_DEPLOY_ATTEMPTS} "
                 "attempts that made no progress (timeouts, lock timeouts or deadlock retries). "
-                "Check database connectivity, load, long-running transactions on the migrated tables, "
-                "and _prisma_migrations ledger state. Raise "
-                f"{PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out, or "
-                f"{MIGRATION_DDL_LOCK_TIMEOUT_ENV_VAR} if they timed out waiting for a table lock."
+                "Check database connectivity, load and _prisma_migrations ledger state. "
+                "If a migration hit SET LOCAL lock_timeout, check for long-running transactions on the migrated tables. "
+                f"Raise {PRISMA_MIGRATE_DEPLOY_TIMEOUT_ENV_VAR} if the attempts timed out."
             )
         finally:
             os.chdir(original_dir)
@@ -1284,8 +1282,8 @@ class ProxyExtrasDBManager:
             if migration_name and _MIGRATION_LOCK_TIMEOUT_MARKER in stderr:
                 ProxyExtrasDBManager._log_migration_lock_holders(migration_name)
                 logger.warning(
-                    "Migration %s timed out waiting for a table lock, rolling its ledger row back and retrying. "
-                    f"Raise {MIGRATION_DDL_LOCK_TIMEOUT_ENV_VAR} if the database needs longer.",
+                    "Migration %s timed out waiting for a table lock after hitting SET LOCAL lock_timeout, rolling "
+                    "its ledger row back and retrying. Check for long-running transactions on the migrated tables.",
                     migration_name,
                 )
                 ProxyExtrasDBManager._v2_roll_back_migration_best_effort(migration_name)

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.108"
+version = "0.4.109"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -30,7 +30,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.108"
+version = "0.4.109"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/crates/framer/src/sse.rs

@@ -1,6 +1,6 @@
 use std::str;
 
-use bytes::{Buf, BufMut, BytesMut};
+use bytes::{Buf, BufMut, Bytes, BytesMut};
 use tokio_util::codec::{Decoder, Encoder};
 
 use crate::SseError;
@@ -29,38 +29,107 @@ impl Decoder for SseCodec {
             return Ok(None);
         }
         while let Some(end) = block_end(src) {
-            let block = src.split_to(end);
-            let pending = lines(&block)
-                .map(|(line, _)| line)
-                .take_while(|line| !line.is_empty())
-                .try_fold(Pending::default(), Pending::apply)?;
-            if let Some(event) = pending.dispatch() {
+            if let Some(event) = event_in(&src.split_to(end))? {
                 return Ok(Some(event));
             }
         }
         Ok(None)
     }
 
-    fn decode_eof(&mut self, _pending: &mut BytesMut) -> Result<Option<SseEvent>, SseError> {
-        Ok(None)
+    /// An event the stream ended inside of is dropped, as the SSE spec requires.
+    fn decode_eof(&mut self, pending: &mut BytesMut) -> Result<Option<SseEvent>, SseError> {
+        self.decode(pending)
     }
 }
 
+pub fn decode_all(wire: &[u8]) -> Result<Vec<SseEvent>, SseError> {
+    let mut codec = SseCodec::default();
+    let mut buffer = BytesMut::from(wire);
+    let mut events = Vec::new();
+    while let Some(event) = codec.decode_eof(&mut buffer)? {
+        events.push(event);
+    }
+    Ok(events)
+}
+
 impl SseCodec {
     fn skip_bom(&mut self, src: &mut BytesMut) -> bool {
-        if self.past_bom {
-            return true;
-        }
-        if src.starts_with(BOM) {
-            src.advance(BOM.len());
-        } else if BOM.starts_with(src) {
-            return false;
-        }
-        self.past_bom = true;
-        true
+        skip_bom(&mut self.past_bom, src)
     }
 }
 
+fn event_in(block: &[u8]) -> Result<Option<SseEvent>, SseError> {
+    Ok(lines(block)
+        .map(|(line, _)| line)
+        .take_while(|line| !line.is_empty())
+        .try_fold(Pending::default(), Pending::apply)?
+        .dispatch())
+}
+
+/// One event together with the exact wire bytes that carried it, for relays that forward a
+/// provider's bytes untouched but still need to see each event go by.
+#[derive(Clone, Debug, PartialEq, Eq)]
+pub struct SseBlock {
+    pub bytes: Bytes,
+    pub event: SseEvent,
+}
+
+#[derive(Clone, Copy, Debug, Default)]
+pub struct SseBlockCodec {
+    past_bom: bool,
+}
+
+impl Decoder for SseBlockCodec {
+    type Item = SseBlock;
+    type Error = SseError;
+
+    fn decode(&mut self, src: &mut BytesMut) -> Result<Option<SseBlock>, SseError> {
+        self.decode_block(src, false)
+    }
+
+    fn decode_eof(&mut self, pending: &mut BytesMut) -> Result<Option<SseBlock>, SseError> {
+        self.decode_block(pending, true)
+    }
+}
+
+impl SseBlockCodec {
+    fn decode_block(
+        &mut self,
+        src: &mut BytesMut,
+        eof: bool,
+    ) -> Result<Option<SseBlock>, SseError> {
+        if !skip_bom(&mut self.past_bom, src) {
+            return Ok(None);
+        }
+        while let Some(end) = block_end(src) {
+            if !eof && end == src.len() && src.ends_with(b"\r") {
+                return Ok(None);
+            }
+            let block = src.split_to(end);
+            if let Some(event) = event_in(&block)? {
+                return Ok(Some(SseBlock {
+                    bytes: block.freeze(),
+                    event,
+                }));
+            }
+        }
+        Ok(None)
+    }
+}
+
+fn skip_bom(past_bom: &mut bool, src: &mut BytesMut) -> bool {
+    if *past_bom {
+        return true;
+    }
+    if src.starts_with(BOM) {
+        src.advance(BOM.len());
+    } else if BOM.starts_with(src) {
+        return false;
+    }
+    *past_bom = true;
+    true
+}
+
 fn block_end(bytes: &[u8]) -> Option<usize> {
     lines(bytes)
         .find(|(line, _)| line.is_empty())
@@ -168,3 +237,179 @@ impl Encoder<SseEvent> for SseCodec {
         Ok(())
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use futures_util::{StreamExt, TryStreamExt, stream};
+    use rstest::rstest;
+
+    use super::*;
+    use crate::frames;
+
+    fn pieces(wire: &[u8], size: usize) -> Vec<Bytes> {
+        wire.chunks(size).map(Bytes::copy_from_slice).collect()
+    }
+
+    async fn blocks(wire: &[u8], size: usize) -> Result<Vec<SseBlock>, SseError> {
+        frames(
+            stream::iter(pieces(wire, size).into_iter().map(Ok::<_, std::io::Error>)),
+            SseBlockCodec::default(),
+        )
+        .try_collect()
+        .await
+    }
+
+    #[rstest]
+    #[case::lf(
+        b": keepalive\n\nevent: delta\ndata: a\ndata: b\n\n",
+        b"event: delta\ndata: a\ndata: b\n\n"
+    )]
+    #[case::crlf(
+        b": keepalive\r\n\r\nevent: delta\r\ndata: a\r\ndata: b\r\n\r\n",
+        b"event: delta\r\ndata: a\r\ndata: b\r\n\r\n"
+    )]
+    #[case::cr(
+        b": keepalive\r\revent: delta\rdata: a\rdata: b\r\r",
+        b"event: delta\rdata: a\rdata: b\r\r"
+    )]
+    #[case::bom(b"\xEF\xBB\xBFdata: \xe2\x82\xac\n\n", b"data: \xe2\x82\xac\n\n")]
+    #[case::empty_data(b"event: ignored\n\ndata:\n\n", b"data:\n\n")]
+    #[case::extensions(
+        b"id: 1\nretry: 10\nvendor: x\ndata: a\n\n",
+        b"id: 1\nretry: 10\nvendor: x\ndata: a\n\n"
+    )]
+    #[tokio::test]
+    async fn blocks_carry_the_wire_bytes_and_the_event_they_dispatch(
+        #[case] wire: &[u8],
+        #[case] expected: &[u8],
+        #[values(1, 3, 4096)] size: usize,
+    ) {
+        let blocks = blocks(wire, size).await.unwrap();
+        let expected_events = decode_all(expected).unwrap();
+        assert_eq!(expected_events.len(), 1);
+        assert_eq!(
+            blocks,
+            vec![SseBlock {
+                bytes: Bytes::copy_from_slice(expected),
+                event: expected_events[0].clone(),
+            }]
+        );
+        assert_eq!(expected_events, decode_all(wire).unwrap());
+    }
+
+    #[rstest]
+    #[case::empty(b"", vec![])]
+    #[case::unterminated(b"data: partial\n", vec![])]
+    #[case::partial_tail(b"data: complete\n\ndata: partial\n", vec![&b"data: complete\n\n"[..]])]
+    #[case::multiple(b"data: first\n\ndata: second\n\n", vec![&b"data: first\n\n"[..], &b"data: second\n\n"[..]])]
+    #[case::trailing_comment(b"data: first\n\n: bye\n\n", vec![&b"data: first\n\n"[..]])]
+    #[tokio::test]
+    async fn blocks_dispatch_complete_events_and_drop_a_cut_tail(
+        #[case] wire: &[u8],
+        #[case] expected: Vec<&[u8]>,
+    ) {
+        let mut blocks = Box::pin(frames(
+            stream::iter(pieces(wire, 1).into_iter().map(Ok::<_, std::io::Error>)),
+            SseBlockCodec::default(),
+        ));
+        for expected in expected {
+            assert_eq!(blocks.next().await.unwrap().unwrap().bytes, expected);
+        }
+        assert!(blocks.next().await.is_none());
+    }
+
+    #[rstest]
+    fn bulk_decode_preserves_every_field() {
+        let events = decode_all(b"id: 1\nretry: 20\nevent: delta\ndata: a\ndata: b\n\n").unwrap();
+        assert_eq!(
+            events,
+            vec![SseEvent {
+                event: Some("delta".into()),
+                data: "a\nb".into(),
+                id: Some("1".into()),
+                retry: Some(20),
+            }]
+        );
+    }
+
+    #[rstest]
+    #[case::only_a_tail(b"data: partial\n", vec![])]
+    #[case::tail_after_an_event(b"data: complete\n\ndata: partial\n", vec!["complete"])]
+    #[case::no_terminator_at_all(b"data: partial", vec![])]
+    fn bulk_decode_drops_the_event_a_stream_ends_inside(
+        #[case] wire: &[u8],
+        #[case] expected: Vec<&str>,
+    ) {
+        let data: Vec<String> = decode_all(wire)
+            .unwrap()
+            .into_iter()
+            .map(|e| e.data)
+            .collect();
+        assert_eq!(data, expected);
+    }
+
+    #[rstest]
+    #[case::trailing_blank_lines(b"data: x\n\n\n\n")]
+    #[case::trailing_comment(b"data: x\n\n: bye\n\n")]
+    #[case::lone_cr_at_eof(b"data: x\r\r")]
+    fn bulk_decode_accepts_a_stream_that_ends_between_events(#[case] wire: &[u8]) {
+        assert_eq!(decode_all(wire).unwrap().len(), 1);
+    }
+
+    #[rstest]
+    fn bulk_decode_rejects_invalid_utf8() {
+        assert!(matches!(
+            decode_all(b"data: \xff\n\n"),
+            Err(SseError::InvalidUtf8(_))
+        ));
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn block_framing_preserves_transport_errors() {
+        let input = stream::iter([
+            Ok(Bytes::from_static(b"data: first\n\n")),
+            Err(std::io::Error::new(
+                std::io::ErrorKind::ConnectionReset,
+                "reset",
+            )),
+        ]);
+        let mut blocks = Box::pin(frames(input, SseBlockCodec::default()));
+        assert_eq!(
+            blocks.next().await.unwrap().unwrap().bytes,
+            Bytes::from_static(b"data: first\n\n")
+        );
+        let Some(Err(SseError::Body(error))) = blocks.next().await else {
+            panic!("expected body error");
+        };
+        let cause = error
+            .get_ref()
+            .unwrap()
+            .downcast_ref::<std::io::Error>()
+            .unwrap();
+        assert_eq!(cause.kind(), std::io::ErrorKind::ConnectionReset);
+        assert!(blocks.next().await.is_none());
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn invalid_utf8_stops_block_framing_after_prior_events() {
+        let mut blocks = Box::pin(frames(
+            stream::iter(
+                pieces(b"data: first\n\ndata: \xff\n\n", 1)
+                    .into_iter()
+                    .map(Ok::<_, std::io::Error>),
+            ),
+            SseBlockCodec::default(),
+        ));
+        assert_eq!(
+            blocks.next().await.unwrap().unwrap().bytes,
+            Bytes::from_static(b"data: first\n\n")
+        );
+        assert!(matches!(
+            blocks.next().await,
+            Some(Err(SseError::InvalidUtf8(_)))
+        ));
+        assert!(blocks.next().await.is_none());
+    }
+}

litellm-rust/crates/framer/tests/sse.rs

@@ -97,12 +97,11 @@ async fn dispatch_follows_the_data_buffer(#[case] wire: &[u8], #[case] expected:
 }
 
 #[rstest]
-#[case::unterminated_single(b"data: partial\n", vec![])]
-#[case::unterminated_tail_after_complete(b"data: complete\n\ndata: unfinished\n", vec![event(None, "complete")])]
 #[case::lone_cr_terminates_at_eof(b"data: x\r\r", vec![event(None, "x")])]
-#[case::lone_cr_line_then_eof(b"data: x\r", vec![])]
+#[case::trailing_blank_lines(b"data: x\n\n\n", vec![event(None, "x")])]
+#[case::trailing_comment(b"data: x\n\n: bye\n\n", vec![event(None, "x")])]
 #[tokio::test]
-async fn eof_dispatches_only_terminated_events(
+async fn eof_between_events_ends_the_stream_cleanly(
     #[case] wire: &[u8],
     #[case] expected: Vec<SseEvent>,
 ) {
@@ -112,6 +111,19 @@ async fn eof_dispatches_only_terminated_events(
     );
 }
 
+#[rstest]
+#[case::unterminated_single(b"data: partial\n", vec![])]
+#[case::unterminated_tail_after_complete(b"data: complete\n\ndata: unfinished\n", vec![event(None, "complete")])]
+#[case::lone_cr_line_then_eof(b"data: x\r", vec![])]
+#[case::partial_bom(b"\xEF\xBB", vec![])]
+#[tokio::test]
+async fn eof_inside_an_event_drops_it_after_the_terminated_events(
+    #[case] wire: &[u8],
+    #[case] expected: Vec<SseEvent>,
+) {
+    assert_eq!(collect(every(wire, 1)).await.unwrap(), expected);
+}
+
 #[rstest]
 #[case::inside_the_first_line(vec![&b"data: a\r"[..], &b"\ndata: b\r\n\r\n"[..]])]
 #[case::inside_the_blank_line(vec![&b"data: a\r\ndata: b\r\n\r"[..], &b"\n"[..]])]

litellm-rust/crates/inference-messages/src/handler.rs

@@ -7,7 +7,7 @@ use litellm_http::transport::Error as TransportError;
 use litellm_llms::base_llm::{
     auth::{Authenticated, resolve_auth},
     messages::{
-        streaming::{ByteStream, StreamDecoder, encode_anthropic_sse},
+        streaming::{self, ByteStream, StreamDecoder},
         transformation::BaseMessagesConfig,
     },
 };
@@ -245,7 +245,7 @@ fn decoded_chunks(
         let Some(event) = events.try_next().await? else {
             return Ok(None);
         };
-        let chunk = encode_anthropic_sse(&event)?;
+        let chunk = streaming::encode(&event)?;
         log_chunk(provider, "client_response", &chunk);
         Ok(Some((chunk, events)))
     })
@@ -271,7 +271,6 @@ fn log_chunk(provider: &str, stage: &str, data: &bytes::Bytes) {
 
 #[cfg(test)]
 mod tests {
-    use litellm_llms::base_llm::messages::streaming::anthropic_sse_event_stream;
     use rstest::rstest;
     use wiremock::{Mock, MockServer, ResponseTemplate, matchers::any};
 
@@ -300,7 +299,7 @@ mod tests {
             .await
             .unwrap();
         let MessagesCallResponse::Stream { mut chunks, .. } =
-            streaming_response(response, Some(anthropic_sse_event_stream), "test")
+            streaming_response(response, Some(streaming::event_stream), "test")
         else {
             panic!("a streaming response returns chunks");
         };

litellm-rust/crates/inference-responses/src/websocket.rs

@@ -2,7 +2,6 @@ use std::{collections::HashMap, sync::Arc, time::Duration};
 
 use futures_util::{SinkExt, StreamExt};
 use litellm_http::websocket::{UpstreamWebSocket, connect_upstream};
-use litellm_llms_types::formats::responses::streaming_websocket::ResponsesWsEventType;
 use tokio::sync::Mutex;
 use tokio_tungstenite::tungstenite::{
     Message,
@@ -12,17 +11,6 @@ use tokio_tungstenite::tungstenite::{
 
 use super::Error;
 
-pub fn is_terminal_event(event_type: &ResponsesWsEventType) -> bool {
-    matches!(
-        event_type,
-        ResponsesWsEventType::ResponseCreated
-            | ResponsesWsEventType::ResponseCompleted
-            | ResponsesWsEventType::ResponseFailed
-            | ResponsesWsEventType::ResponseIncomplete
-            | ResponsesWsEventType::Error
-    )
-}
-
 #[derive(Clone)]
 pub struct ResponsesWebSocketConnection {
     socket: Arc<Mutex<Option<UpstreamWebSocket>>>,

litellm-rust/crates/llms-types/src/formats/chat_completions/streaming.rs

@@ -33,16 +33,82 @@ pub struct ChatCompletionStreamingChoice {
     pub logprobs: Option<Value>,
 }
 
+/// Compatible hosts omit `id`, `created` and `object` on chunks, and Python's
+/// `ModelResponseStream` fills them in, so none of the envelope fields is required here.
 #[macro_rules_attribute::apply(crate::wire_type)]
+#[derive(Default)]
 pub struct ChatCompletionChunk {
-    pub id: String,
-    pub created: u64,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub id: Option<String>,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub created: Option<u64>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub model: Option<String>,
-    pub object: String,
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub object: Option<String>,
+    #[serde(default)]
     pub choices: Vec<ChatCompletionStreamingChoice>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub usage: Option<ChatCompletionsUsage>,
     #[serde(default, skip_serializing_if = "Option::is_none")]
     pub provider_specific_fields: Option<Map<String, Value>>,
+    #[serde(flatten)]
+    pub extra: Map<String, Value>,
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::{Value, json};
+
+    use super::*;
+
+    #[rstest]
+    #[case::openai(json!({
+        "id": "chatcmpl-1", "object": "chat.completion.chunk", "created": 1, "model": "m",
+        "choices": [{"index": 0, "delta": {"role": "assistant", "content": "hi", "future": 1}, "finish_reason": null, "logprobs": null}],
+        "usage": null
+    }))]
+    #[case::usage_only(json!({
+        "id": "chatcmpl-1", "object": "chat.completion.chunk", "created": 1, "model": "m", "choices": [],
+        "usage": {
+            "prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3,
+            "prompt_tokens_details": {"cached_tokens": 0, "cache_creation_tokens": 0, "text_tokens": 1}
+        }
+    }))]
+    #[case::bare_host(json!({"choices": [{"index": 0, "delta": {"content": "hi"}}]}))]
+    #[case::host_extensions(json!({"id": "x", "choices": [], "citations": ["a"], "future": {"k": null}}))]
+    fn chunks_round_trip_without_losing_fields(#[case] wire: Value) {
+        let chunk: ChatCompletionChunk = serde_json::from_value(wire.clone()).unwrap();
+        assert_eq!(serde_json::to_value(chunk).unwrap(), without_nulls(wire));
+    }
+
+    /// Typed optional fields serialize by omission; extension fields keep their nulls.
+    fn without_nulls(value: Value) -> Value {
+        let Value::Object(fields) = value else {
+            return value;
+        };
+        Value::Object(
+            fields
+                .into_iter()
+                .filter(|(_, value)| !value.is_null())
+                .map(|(key, value)| match (key.as_str(), value) {
+                    ("choices", Value::Array(choices)) => (
+                        key,
+                        Value::Array(choices.into_iter().map(without_nulls).collect()),
+                    ),
+                    (_, value) => (key, value),
+                })
+                .collect(),
+        )
+    }
+
+    #[rstest]
+    #[case::choices_not_a_list(json!({"choices": {}}))]
+    #[case::choice_without_delta(json!({"choices": [{"index": 0}]}))]
+    #[case::created_not_a_number(json!({"created": "now", "choices": []}))]
+    #[case::non_object(json!([]))]
+    fn malformed_chunks_are_rejected(#[case] wire: Value) {
+        assert!(serde_json::from_value::<ChatCompletionChunk>(wire).is_err());
+    }
 }

litellm-rust/crates/llms-types/src/formats/messages/streaming.rs

@@ -109,7 +109,9 @@ pub struct MessagesStreamError {
 }
 
 #[macro_rules_attribute::apply(crate::wire_type)]
+#[derive(strum::AsRefStr)]
 #[serde(tag = "type", rename_all = "snake_case")]
+#[strum(serialize_all = "snake_case")]
 pub enum MessagesStreamEvent {
     MessageStart {
         message: MessagesStreamMessage,
@@ -175,6 +177,23 @@ mod tests {
         assert_eq!(serde_json::to_value(event).unwrap(), wire);
     }
 
+    #[rstest]
+    #[case::start(json!({"type": "message_start", "message": {"id": "m", "type": "message", "role": "assistant", "model": "x", "content": [], "stop_reason": null, "stop_sequence": null, "usage": {}}}))]
+    #[case::block_start(json!({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}))]
+    #[case::block_delta(json!({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "a"}}))]
+    #[case::block_stop(json!({"type": "content_block_stop", "index": 0}))]
+    #[case::delta(json!({"type": "message_delta", "delta": {"stop_reason": "end_turn"}}))]
+    #[case::stop(json!({"type": "message_stop"}))]
+    #[case::ping(json!({"type": "ping"}))]
+    #[case::error(json!({"type": "error", "error": {"type": "overloaded_error", "message": "busy"}}))]
+    fn the_event_name_is_the_serialized_type_tag(#[case] wire: Value) {
+        let event: MessagesStreamEvent = serde_json::from_value(wire.clone()).unwrap();
+        assert_eq!(
+            Some(event.as_ref()),
+            wire.get("type").and_then(Value::as_str)
+        );
+    }
+
     #[rstest]
     #[case::invalid_index(json!({"type": "content_block_stop", "index": "zero"}))]
     #[case::missing_delta(json!({"type": "content_block_delta", "index": 0}))]

litellm-rust/crates/llms-types/src/formats/responses/AGENTS.md

@@ -1,7 +1,7 @@
 # rules
 
 - Responses originated at OpenAI and is now offered by other hosts and gateways. Types here describe the format, not OpenAI's deployment of it
-- Served over HTTP and over the WebSocket mode. `ResponsesWsEvent` is a wire event and lives here
+- Served over HTTP and over the WebSocket mode. `ResponsesStreamEvent` and `ResponsesEventType` in `streaming.rs` are the wire event for both
 - `ResponsesWsTransformResult` wraps a provider transformation's output, so it lives in `llms/src/base_llm/responses/`
 - Hosts that accept only part of the format, or add output item types, are handled in `llms/src/<provider>/responses/`. Keep unknown output items passing through instead of rejecting them
 - Responses-to-Chat emulation and model-specific parameter rewriting are transformations, not format rules

litellm-rust/crates/llms-types/src/formats/responses/mod.rs

@@ -1,5 +1,6 @@
 mod output;
 mod response;
+pub mod streaming;
 pub mod streaming_websocket;
 
 pub use output::*;

litellm-rust/crates/llms-types/src/formats/responses/streaming.rs

@@ -0,0 +1,305 @@
+use serde_json::{Map, Value};
+
+/// Every event name on https://developers.openai.com/api/reference/resources/responses/streaming-events
+/// as of 2026-10-11, plus `response.create` from the WebSocket mode. Names OpenAI adds later ride in
+/// `Other` untouched.
+#[derive(
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    strum::AsRefStr,
+    strum::EnumIter,
+    strum::EnumString,
+    strum::Display,
+    serde_with::DeserializeFromStr,
+    serde_with::SerializeDisplay,
+)]
+#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
+#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))]
+pub enum ResponsesEventType {
+    #[strum(serialize = "response.create")]
+    ResponseCreate,
+    #[strum(serialize = "response.created")]
+    ResponseCreated,
+    #[strum(serialize = "response.queued")]
+    ResponseQueued,
+    #[strum(serialize = "response.in_progress")]
+    ResponseInProgress,
+    #[strum(serialize = "response.completed")]
+    ResponseCompleted,
+    #[strum(serialize = "response.failed")]
+    ResponseFailed,
+    #[strum(serialize = "response.incomplete")]
+    ResponseIncomplete,
+    #[strum(serialize = "error")]
+    Error,
+    #[strum(serialize = "response.audio.delta")]
+    AudioDelta,
+    #[strum(serialize = "response.audio.done")]
+    AudioDone,
+    #[strum(serialize = "response.audio.transcript.delta")]
+    AudioTranscriptDelta,
+    #[strum(serialize = "response.audio.transcript.done")]
+    AudioTranscriptDone,
+    #[strum(serialize = "response.code_interpreter_call.completed")]
+    CodeInterpreterCallCompleted,
+    #[strum(serialize = "response.code_interpreter_call.in_progress")]
+    CodeInterpreterCallInProgress,
+    #[strum(serialize = "response.code_interpreter_call.interpreting")]
+    CodeInterpreterCallInterpreting,
+    #[strum(serialize = "response.code_interpreter_call_code.delta")]
+    CodeInterpreterCallCodeDelta,
+    #[strum(serialize = "response.code_interpreter_call_code.done")]
+    CodeInterpreterCallCodeDone,
+    #[strum(serialize = "response.compaction.compacting")]
+    CompactionCompacting,
+    #[strum(serialize = "response.content_part.added")]
+    ContentPartAdded,
+    #[strum(serialize = "response.content_part.done")]
+    ContentPartDone,
+    #[strum(serialize = "response.custom_tool_call_input.delta")]
+    CustomToolCallInputDelta,
+    #[strum(serialize = "response.custom_tool_call_input.done")]
+    CustomToolCallInputDone,
+    #[strum(serialize = "response.file_search_call.completed")]
+    FileSearchCallCompleted,
+    #[strum(serialize = "response.file_search_call.in_progress")]
+    FileSearchCallInProgress,
+    #[strum(serialize = "response.file_search_call.searching")]
+    FileSearchCallSearching,
+    #[strum(serialize = "response.function_call_arguments.delta")]
+    FunctionCallArgumentsDelta,
+    #[strum(serialize = "response.function_call_arguments.done")]
+    FunctionCallArgumentsDone,
+    #[strum(serialize = "response.image_generation_call.completed")]
+    ImageGenerationCallCompleted,
+    #[strum(serialize = "response.image_generation_call.generating")]
+    ImageGenerationCallGenerating,
+    #[strum(serialize = "response.image_generation_call.in_progress")]
+    ImageGenerationCallInProgress,
+    #[strum(serialize = "response.image_generation_call.partial_image")]
+    ImageGenerationCallPartialImage,
+    #[strum(serialize = "response.mcp_call.completed")]
+    McpCallCompleted,
+    #[strum(serialize = "response.mcp_call.failed")]
+    McpCallFailed,
+    #[strum(serialize = "response.mcp_call.in_progress")]
+    McpCallInProgress,
+    #[strum(serialize = "response.mcp_call_arguments.delta")]
+    McpCallArgumentsDelta,
+    #[strum(serialize = "response.mcp_call_arguments.done")]
+    McpCallArgumentsDone,
+    #[strum(serialize = "response.mcp_list_tools.completed")]
+    McpListToolsCompleted,
+    #[strum(serialize = "response.mcp_list_tools.failed")]
+    McpListToolsFailed,
+    #[strum(serialize = "response.mcp_list_tools.in_progress")]
+    McpListToolsInProgress,
+    #[strum(serialize = "response.output_item.added")]
+    OutputItemAdded,
+    #[strum(serialize = "response.output_item.done")]
+    OutputItemDone,
+    #[strum(serialize = "response.output_text.annotation.added")]
+    OutputTextAnnotationAdded,
+    #[strum(serialize = "response.output_text.delta")]
+    OutputTextDelta,
+    #[strum(serialize = "response.output_text.done")]
+    OutputTextDone,
+    #[strum(serialize = "response.reasoning_summary_part.added")]
+    ReasoningSummaryPartAdded,
+    #[strum(serialize = "response.reasoning_summary_part.done")]
+    ReasoningSummaryPartDone,
+    #[strum(serialize = "response.reasoning_summary_text.delta")]
+    ReasoningSummaryTextDelta,
+    #[strum(serialize = "response.reasoning_summary_text.done")]
+    ReasoningSummaryTextDone,
+    #[strum(serialize = "response.reasoning_text.delta")]
+    ReasoningTextDelta,
+    #[strum(serialize = "response.reasoning_text.done")]
+    ReasoningTextDone,
+    #[strum(serialize = "response.refusal.delta")]
+    RefusalDelta,
+    #[strum(serialize = "response.refusal.done")]
+    RefusalDone,
+    #[strum(serialize = "response.shell_call_command.added")]
+    ShellCallCommandAdded,
+    #[strum(serialize = "response.shell_call_command.delta")]
+    ShellCallCommandDelta,
+    #[strum(serialize = "response.shell_call_command.done")]
+    ShellCallCommandDone,
+    #[strum(serialize = "response.shell_call_output_content.delta")]
+    ShellCallOutputContentDelta,
+    #[strum(serialize = "response.shell_call_output_content.done")]
+    ShellCallOutputContentDone,
+    #[strum(serialize = "response.web_search_call.completed")]
+    WebSearchCallCompleted,
+    #[strum(serialize = "response.web_search_call.in_progress")]
+    WebSearchCallInProgress,
+    #[strum(serialize = "response.web_search_call.searching")]
+    WebSearchCallSearching,
+    #[strum(default, transparent)]
+    Other(String),
+}
+
+impl ResponsesEventType {
+    /// The last event of a response. `response.completed`, `response.incomplete` and
+    /// `response.failed` each close the response and carry its final body; `error` aborts it.
+    pub fn is_terminal(&self) -> bool {
+        matches!(
+            self,
+            Self::ResponseCompleted | Self::ResponseFailed | Self::ResponseIncomplete | Self::Error
+        )
+    }
+
+    /// A terminal event after which the response is not a success.
+    pub fn is_failure(&self) -> bool {
+        matches!(
+            self,
+            Self::ResponseFailed | Self::ResponseIncomplete | Self::Error
+        )
+    }
+}
+
+#[macro_rules_attribute::apply(crate::wire_type)]
+pub struct ResponsesStreamEvent {
+    #[serde(rename = "type")]
+    pub event_type: ResponsesEventType,
+    #[serde(flatten)]
+    pub data: Map<String, Value>,
+}
+
+impl ResponsesStreamEvent {
+    pub fn model(&self) -> Option<&str> {
+        let model = self.data.get("model").and_then(Value::as_str);
+        if model.is_some() {
+            return model;
+        }
+        self.data
+            .get("response")
+            .and_then(Value::as_object)
+            .and_then(|response| response.get("model"))
+            .and_then(Value::as_str)
+    }
+
+    pub fn is_response_create(&self) -> bool {
+        self.event_type == ResponsesEventType::ResponseCreate
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+    use serde_json::{Value, json};
+    use strum::IntoEnumIterator;
+
+    use super::*;
+
+    #[rstest]
+    fn every_named_event_type_round_trips() {
+        let named: Vec<_> = ResponsesEventType::iter()
+            .filter(|event_type| !matches!(event_type, ResponsesEventType::Other(_)))
+            .collect();
+        assert!(named.len() > 50);
+        for event_type in named {
+            let wire = event_type.to_string();
+            assert_eq!(wire, event_type.as_ref());
+            assert_eq!(serde_json::to_value(&event_type).unwrap(), json!(wire));
+            assert_eq!(wire.parse::<ResponsesEventType>().unwrap(), event_type);
+            assert_eq!(
+                serde_json::from_value::<ResponsesEventType>(json!(wire)).unwrap(),
+                event_type
+            );
+        }
+    }
+
+    #[rstest]
+    #[case::unknown("future.event", ResponsesEventType::Other("future.event".into()))]
+    #[case::empty("", ResponsesEventType::Other(String::new()))]
+    #[case::case_sensitive(
+        "Response.Completed",
+        ResponsesEventType::Other("Response.Completed".into())
+    )]
+    #[case::escaped("future\"\\\n", ResponsesEventType::Other("future\"\\\n".into()))]
+    fn unknown_event_types_round_trip_through_other(
+        #[case] wire: &str,
+        #[case] expected: ResponsesEventType,
+    ) {
+        assert_eq!(
+            serde_json::from_value::<ResponsesEventType>(json!(wire)).unwrap(),
+            expected
+        );
+        assert_eq!(serde_json::to_value(&expected).unwrap(), json!(wire));
+        assert_eq!(expected.as_ref(), wire);
+        assert_eq!(expected.to_string(), wire);
+    }
+
+    /// Python's `ResponsesAPIStreamingIterator` stores the response on `response.completed`,
+    /// `response.incomplete` and `response.failed`, and raises on `response.failed` and `error`.
+    #[rstest]
+    #[case::completed(ResponsesEventType::ResponseCompleted, true, false)]
+    #[case::incomplete(ResponsesEventType::ResponseIncomplete, true, true)]
+    #[case::failed(ResponsesEventType::ResponseFailed, true, true)]
+    #[case::error(ResponsesEventType::Error, true, true)]
+    #[case::create(ResponsesEventType::ResponseCreate, false, false)]
+    #[case::created(ResponsesEventType::ResponseCreated, false, false)]
+    #[case::in_progress(ResponsesEventType::ResponseInProgress, false, false)]
+    #[case::delta(ResponsesEventType::OutputTextDelta, false, false)]
+    #[case::item_done(ResponsesEventType::OutputItemDone, false, false)]
+    #[case::unknown(ResponsesEventType::Other("response.future".into()), false, false)]
+    #[case::tool_failed(ResponsesEventType::McpCallFailed, false, false)]
+    fn terminal_and_failure_events_match_the_python_iterator(
+        #[case] event_type: ResponsesEventType,
+        #[case] terminal: bool,
+        #[case] failure: bool,
+    ) {
+        assert_eq!(event_type.is_terminal(), terminal);
+        assert_eq!(event_type.is_failure(), failure);
+    }
+
+    #[rstest]
+    #[case::number("17")]
+    #[case::boolean("true")]
+    #[case::null("null")]
+    #[case::array("[]")]
+    #[case::object("{}")]
+    fn event_types_reject_non_strings(#[case] wire: &str) {
+        assert!(serde_json::from_str::<ResponsesEventType>(wire).is_err());
+    }
+
+    #[rstest]
+    #[case::text(json!({"type":"response.output_text.delta","delta":"hello","sequence_number":3,"future":null}))]
+    #[case::tool(json!({"type":"response.function_call_arguments.delta","delta":"{\"x\":","item_id":"fc_1","output_index":0}))]
+    #[case::unknown(json!({"type":"future.event","payload":{"items":[1,null]},"future":true}))]
+    #[case::terminal(json!({"type":"response.completed","response":{"id":"r","output":[],"future":null}}))]
+    fn events_preserve_every_payload_field(#[case] wire: Value) {
+        let event: ResponsesStreamEvent = serde_json::from_value(wire.clone()).unwrap();
+        assert_eq!(serde_json::to_value(event).unwrap(), wire);
+    }
+
+    #[rstest]
+    #[case::missing_type(json!({"delta":"hello"}))]
+    #[case::numeric_type(json!({"type":7}))]
+    #[case::null_type(json!({"type":null}))]
+    #[case::non_object(json!([]))]
+    fn malformed_events_are_rejected(#[case] wire: Value) {
+        assert!(serde_json::from_value::<ResponsesStreamEvent>(wire).is_err());
+    }
+
+    #[rstest]
+    #[case::flat(json!({"type":"response.create","model":"model-a"}), Some("model-a"))]
+    #[case::nested(json!({"type":"response.create","response":{"model":"model-b"}}), Some("model-b"))]
+    #[case::none(json!({"type":"response.created","response":{}}), None)]
+    fn model_reads_flat_and_nested_shapes(#[case] wire: Value, #[case] expected: Option<&str>) {
+        let event: ResponsesStreamEvent = serde_json::from_value(wire).unwrap();
+        assert_eq!(event.model(), expected);
+    }
+
+    #[cfg(feature = "schema")]
+    #[rstest]
+    fn event_type_schema_accepts_unknown_strings() {
+        let schema = schemars::schema_for!(ResponsesEventType).to_value();
+        assert_eq!(schema.get("type"), Some(&json!("string")));
+        assert!(schema.get("enum").is_none());
+    }
+}

litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs

@@ -2,68 +2,6 @@ use super::ResponsesOutputItem;
 use crate::recognized::Recognized;
 use serde_json::{Map, Value};
 
-#[derive(
-    Clone,
-    Debug,
-    PartialEq,
-    Eq,
-    strum::AsRefStr,
-    strum::EnumString,
-    strum::Display,
-    serde_with::DeserializeFromStr,
-    serde_with::SerializeDisplay,
-)]
-#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
-#[cfg_attr(feature = "schema", schemars(with = "String"))]
-pub enum ResponsesWsEventType {
-    #[strum(serialize = "response.create")]
-    ResponseCreate,
-    #[strum(serialize = "response.created")]
-    ResponseCreated,
-    #[strum(serialize = "response.completed")]
-    ResponseCompleted,
-    #[strum(serialize = "response.failed")]
-    ResponseFailed,
-    #[strum(serialize = "response.incomplete")]
-    ResponseIncomplete,
-    #[strum(serialize = "error")]
-    Error,
-    #[strum(default, transparent)]
-    Other(String),
-}
-
-impl ResponsesWsEventType {
-    pub fn as_str(&self) -> &str {
-        self.as_ref()
-    }
-}
-
-#[macro_rules_attribute::apply(crate::wire_type)]
-pub struct ResponsesWsEvent {
-    #[serde(rename = "type")]
-    pub event_type: ResponsesWsEventType,
-    #[serde(flatten)]
-    pub data: Map<String, Value>,
-}
-
-impl ResponsesWsEvent {
-    pub fn model(&self) -> Option<&str> {
-        let model = self.data.get("model").and_then(Value::as_str);
-        if model.is_some() {
-            return model;
-        }
-        self.data
-            .get("response")
-            .and_then(Value::as_object)
-            .and_then(|response| response.get("model"))
-            .and_then(Value::as_str)
-    }
-
-    pub fn is_response_create(&self) -> bool {
-        self.event_type == ResponsesWsEventType::ResponseCreate
-    }
-}
-
 #[macro_rules_attribute::apply(crate::wire_type)]
 #[derive(Eq)]
 pub struct ResponsesErrorFrame {
@@ -112,8 +50,7 @@ mod tests {
     use crate::formats::responses::ResponsesOutputItem;
 
     use crate::{
-        formats::responses::streaming_websocket::{ResponsesEventResponse, ResponsesWsEventType},
-        recognized::Recognized,
+        formats::responses::streaming_websocket::ResponsesEventResponse, recognized::Recognized,
         test_support::*,
     };
 
@@ -134,66 +71,6 @@ mod tests {
         );
     }
 
-    #[rstest]
-    #[case::flat(serde_json::json!({"type":"response.create","model":"model-a"}), Some("model-a"))]
-    #[case::nested(serde_json::json!({"type":"response.create","response":{"model":"model-b"}}), Some("model-b"))]
-    fn model_reads_flat_and_nested_create_shapes(
-        #[case] payload: serde_json::Value,
-        #[case] expected: Option<&str>,
-    ) {
-        let event: ResponsesWsEvent = serde_json::from_value(payload).expect("valid event");
-        assert_eq!(event.model(), expected);
-    }
-
-    #[rstest]
-    #[case::create("response.create", ResponsesWsEventType::ResponseCreate)]
-    #[case::created("response.created", ResponsesWsEventType::ResponseCreated)]
-    #[case::completed("response.completed", ResponsesWsEventType::ResponseCompleted)]
-    #[case::failed("response.failed", ResponsesWsEventType::ResponseFailed)]
-    #[case::incomplete("response.incomplete", ResponsesWsEventType::ResponseIncomplete)]
-    #[case::error("error", ResponsesWsEventType::Error)]
-    #[case::unknown(
-        "response.output_text.delta",
-        ResponsesWsEventType::Other("response.output_text.delta".to_string())
-    )]
-    #[case::empty("", ResponsesWsEventType::Other(String::new()))]
-    #[case::case_sensitive(
-        "Response.Completed",
-        ResponsesWsEventType::Other("Response.Completed".into())
-    )]
-    #[case::escaped("future\"\\\n", ResponsesWsEventType::Other("future\"\\\n".into()))]
-    fn websocket_event_type_round_trips(
-        #[case] wire: &str,
-        #[case] expected: ResponsesWsEventType,
-    ) {
-        let serialized = serde_json::to_string(&expected).unwrap();
-        assert_eq!(serialized, serde_json::to_string(wire).unwrap());
-        assert_eq!(
-            serde_json::from_str::<ResponsesWsEventType>(&serialized).unwrap(),
-            expected
-        );
-    }
-
-    #[rstest]
-    #[case::number("17")]
-    #[case::boolean("true")]
-    #[case::null("null")]
-    #[case::array("[]")]
-    #[case::object("{}")]
-    fn websocket_event_type_rejects_non_strings(#[case] wire: &str) {
-        assert!(serde_json::from_str::<ResponsesWsEventType>(wire).is_err());
-    }
-
-    #[cfg(feature = "schema")]
-    #[rstest]
-    fn websocket_event_type_schema_is_open_string() {
-        let schema = schemars::schema_for!(ResponsesWsEventType);
-        assert_eq!(
-            schema.to_value().get("type"),
-            Some(&serde_json::json!("string"))
-        );
-    }
-
     #[rstest]
     fn nested_event_response_exposes_typed_output_and_preserves_extensions() {
         let wire = json!({

litellm-rust/crates/llms/AGENTS.md

@@ -32,6 +32,13 @@
   - Provider-wide docs (auth, errors, regions): `src/<provider>/`
   - One host's endpoint docs: `src/<provider>/<format>/`
 
+## Streaming
+
+- Each format's `src/base_llm/<format>/streaming.rs` exposes the same three names: `event_stream` (bytes to typed events), `decode` (a whole stream to typed events) and `encode` (one typed event to the SSE frame LiteLLM's proxy sends clients). Callers use the module path, never a vendor prefix
+- SSE framing, `[DONE]` and the generic helpers live in `src/base_llm/sse.rs`. Messages frames are `event:` plus `data:`; Chat Completions and Responses frames are bare `data:`, as Python's proxy writes them
+- Decoding is strict: a malformed event, or `[DONE]` on a Messages stream, is `InvalidResponse` and ends the stream. Python never fails a stream this way; the divergence is deliberate and `src/base_llm/sse.rs` lists what Python does per format. A stream cut inside an event drops that event, as the SSE spec requires
+- What ends or fails a stream belongs on the format's event type (`is_terminal`, `is_failure`). The cache's success check in `inference/src/caching.rs` still matches type strings; move it onto the typed events rather than adding new string matches
+
 ## Python pairs
 
 - Python paths identify counterparts but do not dictate Rust module names or class hierarchy. Preserve behavior and concepts, not structure

litellm-rust/crates/llms/src/anthropic/chat/transformation.rs

@@ -30,7 +30,7 @@ use crate::{
                 Unsupported, ValidatedEnvironment, unsupported_message, unsupported_param,
             },
         },
-        messages::streaming::anthropic_sse_event_stream,
+        messages::streaming::event_stream as messages_event_stream,
     },
 };
 
@@ -196,7 +196,7 @@ impl BaseConfig for AnthropicConfig {
 
     fn model_response_iterator(&self, shape: StreamShape) -> Option<ChatStream> {
         Some(ChatStream::new(
-            anthropic_sse_event_stream,
+            messages_event_stream,
             ModelResponseIterator::new(shape),
         ))
     }

litellm-rust/crates/llms/src/base_llm/chat/streaming.rs

@@ -1,15 +1,32 @@
 use std::collections::HashMap;
 
+use bytes::Bytes;
 use futures_util::{StreamExt, stream::BoxStream};
 use litellm_llms_types::formats::chat_completions::ChatCompletionChunk;
 
 use crate::{
     Error,
-    base_llm::base_model_iterator::{ByteStream, StreamError, StreamTransformer, transform_stream},
+    base_llm::{
+        base_model_iterator::{ByteStream, StreamError, StreamTransformer, transform_stream},
+        sse::{self, is_done},
+    },
 };
 
 pub type ChatChunkStream = BoxStream<'static, Result<ChatCompletionChunk, Error>>;
 
+/// OpenAI-compatible SSE: bare `data:` frames closed by `data: [DONE]`.
+pub fn event_stream(bytes: ByteStream) -> ChatChunkStream {
+    sse::event_stream(bytes, is_done)
+}
+
+pub fn decode(wire: &[u8]) -> Result<Vec<ChatCompletionChunk>, Error> {
+    sse::decode(wire, is_done)
+}
+
+pub fn encode(chunk: &ChatCompletionChunk) -> Result<Bytes, Error> {
+    sse::encode_data(chunk)
+}
+
 /// What Python's `map_openai_params` decides about the stream and `completion`
 /// hands to `ModelResponseIterator`: it is settled while the request is built,
 /// never re-derived from the body.
@@ -52,3 +69,109 @@ impl ChatStream {
         (self.run)(bytes)
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use futures_util::{StreamExt, TryStreamExt, stream};
+    use rstest::rstest;
+    use serde_json::{Value, json};
+
+    use super::*;
+
+    fn in_pieces(wire: &[u8], size: usize) -> ByteStream {
+        let pieces: Vec<Bytes> = wire.chunks(size).map(Bytes::copy_from_slice).collect();
+        stream::iter(pieces.into_iter().map(Ok)).boxed()
+    }
+
+    #[rstest]
+    #[case::role(json!({"id":"c","object":"chat.completion.chunk","created":1,"model":"m","choices":[{"index":0,"delta":{"role":"assistant","content":""}}]}))]
+    #[case::text(json!({"id":"c","choices":[{"index":0,"delta":{"content":"hi\nthere"}}]}))]
+    #[case::tool(json!({"id":"c","choices":[{"index":0,"delta":{"tool_calls":[{"index":0,"id":"t","type":"function","function":{"name":"f","arguments":"{\"x\":"}}]}}]}))]
+    #[case::finish(json!({"id":"c","choices":[{"index":0,"delta":{},"finish_reason":"stop"}]}))]
+    #[case::usage(json!({"id":"c","choices":[],"usage":{"prompt_tokens":1,"completion_tokens":2,"total_tokens":3,"prompt_tokens_details":{"cached_tokens":0,"cache_creation_tokens":0,"text_tokens":1}}}))]
+    #[case::host_extension(json!({"id":"c","choices":[],"citations":[{"url":"u"}]}))]
+    fn encoded_chunks_are_bare_data_frames_that_decode_back(#[case] wire: Value) {
+        let chunk: ChatCompletionChunk = serde_json::from_value(wire.clone()).unwrap();
+        let encoded = encode(&chunk).unwrap();
+        assert!(encoded.starts_with(b"data: "));
+        assert!(encoded.ends_with(b"\n\n"));
+        assert_eq!(
+            serde_json::from_slice::<Value>(&encoded[b"data: ".len()..encoded.len() - 2]).unwrap(),
+            wire
+        );
+        assert_eq!(decode(&encoded).unwrap(), vec![chunk]);
+    }
+
+    #[rstest]
+    #[case::lf("\n")]
+    #[case::crlf("\r\n")]
+    #[case::cr("\r")]
+    #[tokio::test]
+    async fn streams_split_anywhere_skip_comments_and_done(
+        #[case] newline: &str,
+        #[values(1, 5, 4096)] size: usize,
+    ) {
+        let wire = format!(
+            ": keepalive{newline}{newline}data: {{\"id\":\"c\",\"choices\":[{{\"index\":0,\"delta\":{{\"content\":\"a\"}}}}]}}{newline}{newline}data:{{\"id\":\"c\",\"choices\":[{{\"index\":0,\"delta\":{{}},\"finish_reason\":\"stop\"}}]}}{newline}{newline}data: [DONE]{newline}{newline}"
+        );
+        let streamed = event_stream(in_pieces(wire.as_bytes(), size))
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(streamed, decode(wire.as_bytes()).unwrap());
+        assert_eq!(streamed.len(), 2);
+        assert_eq!(streamed[0].choices[0].delta.content.as_deref(), Some("a"));
+        assert_eq!(
+            streamed[1].choices[0].finish_reason.as_deref(),
+            Some("stop")
+        );
+    }
+
+    #[rstest]
+    #[case::spaced(b"data: [DONE]\n\n")]
+    #[case::unspaced(b"data:[DONE]\n\n")]
+    #[case::trailing_space(b"data: [DONE] \n\n")]
+    fn every_done_spelling_closes_the_stream_without_a_chunk(#[case] wire: &[u8]) {
+        assert_eq!(decode(wire).unwrap(), vec![]);
+    }
+
+    #[rstest]
+    #[case::invalid_json(b"data: {\n\n")]
+    #[case::choices_not_a_list(b"data: {\"choices\":{}}\n\n")]
+    #[case::invalid_utf8(b"data: \xff\n\n")]
+    #[tokio::test]
+    async fn malformed_streams_are_invalid_responses(#[case] wire: &[u8]) {
+        assert!(matches!(decode(wire), Err(Error::InvalidResponse(_))));
+        let streamed = event_stream(in_pieces(wire, 1)).collect::<Vec<_>>().await;
+        assert!(matches!(
+            streamed.last(),
+            Some(Err(Error::InvalidResponse(_))) | None
+        ));
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn a_body_error_ends_the_stream_after_delivered_chunks() {
+        let mut chunks = event_stream(
+            stream::iter([
+                Ok(Bytes::from_static(
+                    b"data: {\"id\":\"c\",\"choices\":[]}\n\n",
+                )),
+                Err(std::io::Error::new(
+                    std::io::ErrorKind::ConnectionReset,
+                    "reset",
+                )),
+            ])
+            .boxed(),
+        );
+        assert_eq!(
+            chunks.next().await.unwrap().unwrap().id.as_deref(),
+            Some("c")
+        );
+        assert!(matches!(
+            chunks.next().await,
+            Some(Err(Error::InvalidResponse(_)))
+        ));
+        assert!(chunks.next().await.is_none());
+    }
+}

litellm-rust/crates/llms/src/base_llm/messages/streaming.rs

@@ -1,38 +1,32 @@
 use bytes::Bytes;
-use futures_util::{StreamExt, stream::BoxStream};
-use litellm_framer::{frames, sse::SseCodec};
 use litellm_llms_types::formats::messages::streaming::MessagesStreamEvent;
 
-use crate::Error;
 pub use crate::base_llm::base_model_iterator::ByteStream;
+use crate::{Error, base_llm::sse};
 
-pub type EventStream = BoxStream<'static, Result<MessagesStreamEvent, Error>>;
+pub type EventStream = sse::EventStream<MessagesStreamEvent>;
 pub type StreamDecoder = fn(ByteStream) -> EventStream;
 
-pub fn anthropic_sse_event_stream(bytes: ByteStream) -> EventStream {
-    Box::pin(frames(bytes, SseCodec::default()).map(|event| {
-        let event = event.map_err(|error| {
-            Error::InvalidResponse(crate::ErrorDetail::failed("stream framing", error))
-        })?;
-        serde_json::from_str(&event.data).map_err(|error| {
-            Error::InvalidResponse(crate::ErrorDetail::invalid("Anthropic stream event", error))
-        })
-    }))
+fn no_control_frames(_: &litellm_framer::sse::SseEvent) -> bool {
+    false
 }
 
-pub fn encode_anthropic_sse(event: &MessagesStreamEvent) -> Result<Bytes, Error> {
-    let data = serde_json::to_value(event).map_err(|error| {
-        Error::InvalidResponse(crate::ErrorDetail::invalid("Anthropic stream event", error))
-    })?;
-    let name = data
-        .get("type")
-        .and_then(serde_json::Value::as_str)
-        .ok_or_else(|| {
-            Error::InvalidResponse(
-                "Anthropic stream event is invalid: stream event has no type".into(),
-            )
-        })?;
-    Ok(Bytes::from(format!("event: {name}\ndata: {data}\n\n")))
+pub fn event_stream(bytes: ByteStream) -> EventStream {
+    sse::event_stream(bytes, no_control_frames)
+}
+
+pub fn decode(wire: &[u8]) -> Result<Vec<MessagesStreamEvent>, Error> {
+    sse::decode(wire, no_control_frames)
+}
+
+/// Anthropic names every frame after the event's `type`, and LiteLLM's proxy does the same
+/// when it re-emits translated events.
+pub fn encode(event: &MessagesStreamEvent) -> Result<Bytes, Error> {
+    let data = serde_json::to_string(event).map_err(sse::invalid)?;
+    Ok(Bytes::from(format!(
+        "event: {}\ndata: {data}\n\n",
+        event.as_ref()
+    )))
 }
 
 #[cfg(test)]
@@ -41,6 +35,7 @@ mod tests {
     use litellm_llms_types::formats::messages::streaming::{
         MessagesContentBlockDelta, MessagesStreamUsage,
     };
+    use rstest::rstest;
     use serde_json::json;
 
     use super::*;
@@ -48,51 +43,11 @@ mod tests {
     const TEXT_DELTA: &str =
         r#"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"hello"}}"#;
 
-    fn in_pieces(wire: &[u8]) -> ByteStream {
-        let pieces: Vec<Bytes> = wire.chunks(3).map(Bytes::copy_from_slice).collect();
+    fn in_pieces(wire: &[u8], size: usize) -> ByteStream {
+        let pieces: Vec<Bytes> = wire.chunks(size).map(Bytes::copy_from_slice).collect();
         stream::iter(pieces.into_iter().map(Ok)).boxed()
     }
 
-    #[tokio::test]
-    async fn sse_frames_split_anywhere_decode_into_typed_events() {
-        let wire = format!("event: content_block_delta\ndata: {TEXT_DELTA}\n\n");
-        let events = anthropic_sse_event_stream(in_pieces(wire.as_bytes()))
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-
-        assert_eq!(
-            events,
-            vec![MessagesStreamEvent::ContentBlockDelta {
-                index: 0,
-                delta: MessagesContentBlockDelta::TextDelta {
-                    text: "hello".into(),
-                },
-            }]
-        );
-    }
-
-    #[tokio::test]
-    async fn decodes_citations_delta_events() {
-        let wire = concat!(
-            "event: content_block_delta\n",
-            r#"data: {"type":"content_block_delta","index":0,"delta":{"type":"citations_delta","citation":{"type":"char_location"}}}"#,
-            "\n\n",
-        );
-        let events = anthropic_sse_event_stream(in_pieces(wire.as_bytes()))
-            .try_collect::<Vec<_>>()
-            .await
-            .unwrap();
-
-        assert!(matches!(
-            events.as_slice(),
-            [MessagesStreamEvent::ContentBlockDelta {
-                delta: MessagesContentBlockDelta::Citations { .. },
-                ..
-            }]
-        ));
-    }
-
     fn events() -> Vec<MessagesStreamEvent> {
         vec![
             MessagesStreamEvent::Ping,
@@ -110,27 +65,80 @@ mod tests {
         ]
     }
 
-    #[tokio::test]
-    async fn encoded_events_decode_back_to_themselves() {
-        let wire = events()
-            .iter()
-            .map(encode_anthropic_sse)
-            .collect::<Result<Vec<_>, _>>()
-            .unwrap();
+    fn wire(newline: &str) -> String {
+        format!(
+            "event: content_block_delta{newline}data: {TEXT_DELTA}{newline}{newline}: keepalive{newline}{newline}event: message_stop{newline}data: {{\"type\":\"message_stop\"}}{newline}{newline}"
+        )
+    }
 
-        let decoded = anthropic_sse_event_stream(stream::iter(wire.into_iter().map(Ok)).boxed())
+    #[rstest]
+    #[case::lf("\n")]
+    #[case::crlf("\r\n")]
+    #[case::cr("\r")]
+    #[tokio::test]
+    async fn frames_split_anywhere_decode_into_typed_events(
+        #[case] newline: &str,
+        #[values(1, 3, 4096)] size: usize,
+    ) {
+        let wire = wire(newline);
+        let expected = vec![
+            MessagesStreamEvent::ContentBlockDelta {
+                index: 0,
+                delta: MessagesContentBlockDelta::TextDelta {
+                    text: "hello".into(),
+                },
+            },
+            MessagesStreamEvent::MessageStop { usage: None },
+        ];
+        let streamed = event_stream(in_pieces(wire.as_bytes(), size))
             .try_collect::<Vec<_>>()
             .await
             .unwrap();
+        assert_eq!(streamed, expected);
+        assert_eq!(decode(wire.as_bytes()).unwrap(), expected);
+    }
 
-        assert_eq!(decoded, events());
+    #[tokio::test]
+    async fn decodes_citations_delta_events() {
+        let wire = concat!(
+            "event: content_block_delta\n",
+            r#"data: {"type":"content_block_delta","index":0,"delta":{"type":"citations_delta","citation":{"type":"char_location"}}}"#,
+            "\n\n",
+        );
+        let events = event_stream(in_pieces(wire.as_bytes(), 3))
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert!(matches!(
+            events.as_slice(),
+            [MessagesStreamEvent::ContentBlockDelta {
+                delta: MessagesContentBlockDelta::Citations { .. },
+                ..
+            }]
+        ));
+    }
+
+    #[tokio::test]
+    async fn encoded_events_decode_back_to_themselves() {
+        let wire: Vec<u8> = events()
+            .iter()
+            .map(encode)
+            .collect::<Result<Vec<_>, _>>()
+            .unwrap()
+            .into_iter()
+            .flatten()
+            .collect();
+        let streamed = event_stream(in_pieces(&wire, 3))
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(streamed, events());
+        assert_eq!(decode(&wire).unwrap(), events());
     }
 
     #[test]
     fn an_event_is_named_by_its_type() {
-        let encoded =
-            encode_anthropic_sse(&MessagesStreamEvent::MessageStop { usage: None }).unwrap();
-
+        let encoded = encode(&MessagesStreamEvent::MessageStop { usage: None }).unwrap();
         assert_eq!(
             encoded,
             Bytes::from(format!(
@@ -139,4 +147,68 @@ mod tests {
             ))
         );
     }
+
+    #[rstest]
+    #[case::json(b"data: {\n\n")]
+    #[case::unknown_event(b"data: {\"type\":\"future_event\"}\n\n")]
+    #[case::malformed_tool(b"data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":3}}\n\n")]
+    #[case::utf8(b"data: \xff\n\n")]
+    #[case::done_is_not_an_anthropic_frame(b"data: [DONE]\n\n")]
+    #[tokio::test]
+    async fn malformed_streams_are_invalid_responses(#[case] wire: &[u8]) {
+        assert!(matches!(decode(wire), Err(Error::InvalidResponse(_))));
+        let streamed = event_stream(in_pieces(wire, 1)).collect::<Vec<_>>().await;
+        assert!(matches!(
+            streamed.last(),
+            Some(Err(Error::InvalidResponse(_))) | None
+        ));
+    }
+
+    #[rstest]
+    fn decoding_keeps_tool_deltas_and_block_extensions() {
+        let wire = concat!(
+            "data: {\"type\":\"content_block_start\",\"index\":0,\"content_block\":{\"type\":\"tool_use\",\"id\":\"t\",\"name\":\"lookup\",\"input\":{},\"future\":null}}\n\n",
+            "data: {\"type\":\"content_block_delta\",\"index\":0,\"delta\":{\"type\":\"input_json_delta\",\"partial_json\":\"{\\\"x\\\":\"}}\n\n",
+        );
+        let decoded = decode(wire.as_bytes()).unwrap();
+        let [
+            MessagesStreamEvent::ContentBlockStart { content_block, .. },
+            MessagesStreamEvent::ContentBlockDelta {
+                delta: MessagesContentBlockDelta::InputJsonDelta { partial_json },
+                ..
+            },
+        ] = decoded.as_slice()
+        else {
+            panic!("expected tool events");
+        };
+        assert_eq!(
+            content_block.extra.get("future"),
+            Some(&serde_json::Value::Null)
+        );
+        assert_eq!(partial_json, "{\"x\":");
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn a_body_error_ends_the_stream_after_delivered_events() {
+        let mut events = event_stream(
+            stream::iter([
+                Ok(Bytes::from_static(b"data: {\"type\":\"ping\"}\n\n")),
+                Err(std::io::Error::new(
+                    std::io::ErrorKind::ConnectionReset,
+                    "reset",
+                )),
+            ])
+            .boxed(),
+        );
+        assert_eq!(
+            events.next().await.unwrap().unwrap(),
+            MessagesStreamEvent::Ping
+        );
+        assert!(matches!(
+            events.next().await,
+            Some(Err(Error::InvalidResponse(_)))
+        ));
+        assert!(events.next().await.is_none());
+    }
 }

litellm-rust/crates/llms/src/base_llm/mod.rs

@@ -5,3 +5,4 @@ pub mod chat;
 pub mod messages;
 pub mod ocr;
 pub mod responses;
+pub mod sse;

litellm-rust/crates/llms/src/base_llm/responses/mod.rs

@@ -1 +1,2 @@
+pub mod streaming;
 pub mod transformation;

litellm-rust/crates/llms/src/base_llm/responses/streaming.rs

@@ -0,0 +1,135 @@
+use bytes::Bytes;
+use litellm_llms_types::formats::responses::streaming::ResponsesStreamEvent;
+
+pub use crate::base_llm::base_model_iterator::ByteStream;
+use crate::{
+    Error,
+    base_llm::sse::{self, is_done},
+};
+
+pub type EventStream = sse::EventStream<ResponsesStreamEvent>;
+
+pub fn event_stream(bytes: ByteStream) -> EventStream {
+    sse::event_stream(bytes, is_done)
+}
+
+pub fn decode(wire: &[u8]) -> Result<Vec<ResponsesStreamEvent>, Error> {
+    sse::decode(wire, is_done)
+}
+
+/// A bare `data:` frame. Python's proxy serializes every Responses event this way and never
+/// writes an `event:` line, so clients see one shape whichever runtime served them.
+pub fn encode(event: &ResponsesStreamEvent) -> Result<Bytes, Error> {
+    sse::encode_data(event)
+}
+
+#[cfg(test)]
+mod tests {
+    use futures_util::{StreamExt, TryStreamExt, stream};
+    use litellm_llms_types::formats::responses::streaming::ResponsesEventType;
+    use rstest::rstest;
+    use serde_json::{Value, json};
+
+    use super::*;
+
+    fn in_pieces(wire: &[u8], size: usize) -> ByteStream {
+        let pieces: Vec<Bytes> = wire.chunks(size).map(Bytes::copy_from_slice).collect();
+        stream::iter(pieces.into_iter().map(Ok)).boxed()
+    }
+
+    #[rstest]
+    #[case::created(json!({"type":"response.created","response":{"id":"r","output":[]}}))]
+    #[case::text(json!({"type":"response.output_text.delta","delta":"hello\nworld","sequence_number":3}))]
+    #[case::tool(json!({"type":"response.function_call_arguments.delta","delta":"{\"x\":","item_id":"fc_1"}))]
+    #[case::unknown(json!({"type":"future.event","payload":[1,null],"future":{"nested":true}}))]
+    #[case::completed(json!({"type":"response.completed","response":{"id":"r","usage":{"input_tokens":3}}}))]
+    #[case::failed(json!({"type":"response.failed","response":{"error":{"code":"failure"}}}))]
+    #[case::incomplete(json!({"type":"response.incomplete","response":{"incomplete_details":{"reason":"limit"}}}))]
+    #[case::error(json!({"type":"error","error":{"message":"boom"}}))]
+    fn encoded_events_are_bare_data_frames_that_decode_back(#[case] wire: Value) {
+        let event: ResponsesStreamEvent = serde_json::from_value(wire.clone()).unwrap();
+        let encoded = encode(&event).unwrap();
+        assert!(encoded.starts_with(b"data: "));
+        assert!(encoded.ends_with(b"\n\n"));
+        assert_eq!(
+            serde_json::from_slice::<Value>(&encoded[b"data: ".len()..encoded.len() - 2]).unwrap(),
+            wire
+        );
+        assert_eq!(decode(&encoded).unwrap(), vec![event]);
+    }
+
+    #[rstest]
+    #[case::lf("\n")]
+    #[case::crlf("\r\n")]
+    #[case::cr("\r")]
+    #[tokio::test]
+    async fn streams_split_anywhere_skip_comments_and_done(
+        #[case] newline: &str,
+        #[values(1, 5, 4096)] size: usize,
+    ) {
+        let wire = format!(
+            "\u{feff}event: response.created{newline}data: {{\"type\":\"response.created\"}}{newline}{newline}: keepalive{newline}{newline}event: future.event{newline}data: {{\"type\":\"future.event\",\"payload\":null}}{newline}{newline}data: [DONE]{newline}{newline}"
+        );
+        let streamed = event_stream(in_pieces(wire.as_bytes(), size))
+            .try_collect::<Vec<_>>()
+            .await
+            .unwrap();
+        assert_eq!(streamed, decode(wire.as_bytes()).unwrap());
+        assert_eq!(streamed.len(), 2);
+        assert_eq!(streamed[0].event_type, ResponsesEventType::ResponseCreated);
+        assert_eq!(
+            streamed[1].event_type,
+            ResponsesEventType::Other("future.event".into())
+        );
+        assert_eq!(streamed[1].data.get("payload"), Some(&Value::Null));
+    }
+
+    #[rstest]
+    #[case::spaced(b"data: [DONE]\n\n")]
+    #[case::unspaced(b"data:[DONE]\n\n")]
+    #[case::trailing_space(b"data: [DONE] \n\n")]
+    fn every_done_spelling_closes_the_stream_without_an_event(#[case] wire: &[u8]) {
+        assert_eq!(decode(wire).unwrap(), vec![]);
+    }
+
+    #[rstest]
+    #[case::invalid_json(b"data: {\n\n")]
+    #[case::missing_type(b"data: {}\n\n")]
+    #[case::numeric_type(b"data: {\"type\":3}\n\n")]
+    #[case::invalid_utf8(b"data: \xff\n\n")]
+    #[tokio::test]
+    async fn malformed_streams_are_invalid_responses(#[case] wire: &[u8]) {
+        assert!(matches!(decode(wire), Err(Error::InvalidResponse(_))));
+        let streamed = event_stream(in_pieces(wire, 1)).collect::<Vec<_>>().await;
+        assert!(matches!(
+            streamed.last(),
+            Some(Err(Error::InvalidResponse(_))) | None
+        ));
+    }
+
+    #[rstest]
+    #[tokio::test]
+    async fn a_body_error_ends_the_stream_after_delivered_events() {
+        let mut events = event_stream(
+            stream::iter([
+                Ok(Bytes::from_static(
+                    b"data: {\"type\":\"response.created\"}\n\n",
+                )),
+                Err(std::io::Error::new(
+                    std::io::ErrorKind::ConnectionReset,
+                    "reset",
+                )),
+            ])
+            .boxed(),
+        );
+        assert_eq!(
+            events.next().await.unwrap().unwrap().event_type,
+            ResponsesEventType::ResponseCreated
+        );
+        assert!(matches!(
+            events.next().await,
+            Some(Err(Error::InvalidResponse(_)))
+        ));
+        assert!(events.next().await.is_none());
+    }
+}

litellm-rust/crates/llms/src/base_llm/responses/transformation.rs

@@ -1,5 +1,5 @@
 use litellm_llms_types::formats::responses::{
-    ResponsesApiResponse, streaming_websocket::ResponsesWsEvent,
+    ResponsesApiResponse, streaming::ResponsesStreamEvent,
 };
 use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value};
@@ -38,11 +38,11 @@ pub trait BaseResponsesApiConfig: Sync {
 
 #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
 pub struct ResponsesWsTransformResult {
-    pub events: Vec<ResponsesWsEvent>,
+    pub events: Vec<ResponsesStreamEvent>,
 }
 
 impl ResponsesWsTransformResult {
-    pub fn passthrough(event: ResponsesWsEvent) -> Self {
+    pub fn passthrough(event: ResponsesStreamEvent) -> Self {
         Self {
             events: vec![event],
         }
@@ -58,18 +58,18 @@ pub trait ResponsesWebSocketProviderConfig: Sync {
 
     fn transform_ws_request(
         &self,
-        event: &ResponsesWsEvent,
+        event: &ResponsesStreamEvent,
         model: &str,
     ) -> Result<ResponsesWsTransformResult, Error>;
 
     fn transform_ws_response(
         &self,
-        event: &ResponsesWsEvent,
+        event: &ResponsesStreamEvent,
         model: &str,
     ) -> Result<ResponsesWsTransformResult, Error>;
 }
 
-pub fn enforce_model(event: &ResponsesWsEvent, model: &str) -> ResponsesWsEvent {
+pub fn enforce_model(event: &ResponsesStreamEvent, model: &str) -> ResponsesStreamEvent {
     if !event.is_response_create() {
         return event.clone();
     }
@@ -103,7 +103,7 @@ pub fn enforce_model(event: &ResponsesWsEvent, model: &str) -> ResponsesWsEvent
 mod tests {
     use super::*;
 
-    fn event(value: serde_json::Value) -> ResponsesWsEvent {
+    fn event(value: serde_json::Value) -> ResponsesStreamEvent {
         serde_json::from_value(value).expect("valid event")
     }
 

litellm-rust/crates/llms/src/base_llm/sse.rs

@@ -0,0 +1,102 @@
+//! SSE decoding and encoding shared by every format's `streaming` module.
+//!
+//! DIVERGENCE: decoding is strict. An event whose `data` is not a valid event of the format is
+//! `InvalidResponse` and ends the stream, and so is `[DONE]` on a Messages stream. Python
+//! never fails a stream on a malformed event:
+//! - Chat Completions (`BaseModelResponseIterator`) and Responses
+//!   (`BaseResponsesAPIStreamingIterator`) skip the event and keep going
+//! - Messages (`BaseAnthropicMessagesStreamingIterator`) forwards the provider's bytes to the
+//!   client without parsing them
+//! - Anthropic to Chat (`anthropic/chat/handler.py` `ModelResponseIterator`) buffers the invalid
+//!   lines on the guess that one payload was split across them
+//!
+//! Skipping loses content or a tool-call fragment with nothing to show for it, so Rust reports it.
+
+use bytes::Bytes;
+use futures_util::{StreamExt, TryStreamExt, future::ready, stream::BoxStream};
+use litellm_framer::{
+    frames,
+    sse::{SseCodec, SseEvent, decode_all},
+};
+use serde::{Serialize, de::DeserializeOwned};
+
+use crate::{Error, ErrorDetail, base_llm::base_model_iterator::ByteStream};
+
+pub type EventStream<E> = BoxStream<'static, Result<E, Error>>;
+
+/// Which frames are control frames rather than events of the format.
+pub type IsControl = fn(&SseEvent) -> bool;
+
+/// OpenAI's end-of-stream marker on Chat Completions and Responses streams. It closes the
+/// stream and is not an event; Anthropic Messages streams never carry it.
+pub const DONE_FRAME: &[u8] = b"data: [DONE]\n\n";
+
+pub fn is_done(event: &SseEvent) -> bool {
+    event.data.trim() == "[DONE]"
+}
+
+pub fn event_stream<E: DeserializeOwned + Send + 'static>(
+    bytes: ByteStream,
+    is_control: IsControl,
+) -> EventStream<E> {
+    frames(bytes, SseCodec::default())
+        .map_err(framing)
+        .try_filter(move |event| ready(!is_control(event)))
+        .and_then(|event| ready(parse(&event)))
+        .boxed()
+}
+
+/// Decodes a whole stream at once. An event the stream ends inside is dropped.
+pub fn decode<E: DeserializeOwned>(wire: &[u8], is_control: IsControl) -> Result<Vec<E>, Error> {
+    decode_all(wire)
+        .map_err(framing)?
+        .iter()
+        .filter(|event| !is_control(event))
+        .map(parse)
+        .collect()
+}
+
+/// A bare `data:` frame, which is what LiteLLM's proxy sends to clients on OpenAI formats.
+pub fn encode_data<E: Serialize>(event: &E) -> Result<Bytes, Error> {
+    let data = serde_json::to_string(event).map_err(invalid)?;
+    Ok(Bytes::from(format!("data: {data}\n\n")))
+}
+
+fn parse<E: DeserializeOwned>(event: &SseEvent) -> Result<E, Error> {
+    serde_json::from_str(&event.data).map_err(invalid)
+}
+
+pub fn invalid(error: serde_json::Error) -> Error {
+    Error::InvalidResponse(ErrorDetail::invalid("stream event", error))
+}
+
+fn framing(error: litellm_framer::SseError) -> Error {
+    Error::InvalidResponse(ErrorDetail::failed("stream framing", error))
+}
+
+#[cfg(test)]
+mod tests {
+    use rstest::rstest;
+
+    use super::*;
+
+    #[rstest]
+    #[case::exact("[DONE]", true)]
+    #[case::padded(" [DONE] ", true)]
+    #[case::json("{\"type\":\"x\"}", false)]
+    #[case::mentions("[DONE] was sent", false)]
+    #[case::empty("", false)]
+    fn done_is_an_exact_marker(#[case] data: &str, #[case] done: bool) {
+        let event = SseEvent {
+            data: data.into(),
+            ..SseEvent::default()
+        };
+        assert_eq!(is_done(&event), done);
+    }
+
+    #[rstest]
+    fn the_done_frame_we_send_decodes_as_a_control_frame() {
+        let events: Vec<serde_json::Value> = decode(DONE_FRAME, is_done).unwrap();
+        assert!(events.is_empty());
+    }
+}

litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs

@@ -102,7 +102,7 @@ mod tests {
     use litellm_llms_types::formats::messages::streaming::MessagesContentBlockDelta;
 
     use super::*;
-    use crate::base_llm::messages::streaming::anthropic_sse_event_stream;
+    use crate::base_llm::messages::streaming;
 
     fn in_pieces(wire: &[u8]) -> ByteStream {
         let pieces: Vec<Bytes> = wire.chunks(3).map(Bytes::copy_from_slice).collect();
@@ -147,7 +147,7 @@ mod tests {
             .try_collect::<Vec<_>>()
             .await
             .unwrap();
-        let from_sse = anthropic_sse_event_stream(in_pieces(sse.as_bytes()))
+        let from_sse = streaming::event_stream(in_pieces(sse.as_bytes()))
             .try_collect::<Vec<_>>()
             .await
             .unwrap();

litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs

@@ -332,7 +332,7 @@ mod tests {
     use litellm_auth_aws::constants::DEFAULT_BEDROCK_REGION;
 
     use super::*;
-    use crate::base_llm::messages::streaming::encode_anthropic_sse;
+    use crate::base_llm::messages::streaming;
 
     fn event(value: Value) -> MessagesStreamEvent {
         serde_json::from_value(value).unwrap()
@@ -493,7 +493,7 @@ mod tests {
         .boxed();
 
         let sse = bedrock_anthropic_messages_event_stream(bytes)
-            .map_ok(|event| encode_anthropic_sse(&event).unwrap())
+            .map_ok(|event| streaming::encode(&event).unwrap())
             .try_collect::<Vec<_>>()
             .await
             .unwrap()
@@ -508,7 +508,7 @@ mod tests {
             )),
         ]
         .iter()
-        .flat_map(|event| encode_anthropic_sse(event).unwrap())
+        .flat_map(|event| streaming::encode(event).unwrap())
         .collect();
         assert_eq!(sse, expected);
     }

litellm-rust/crates/llms/src/openai/responses/transformation.rs

@@ -1,5 +1,5 @@
 use litellm_llms_types::formats::responses::{
-    ResponsesApiResponse, streaming_websocket::ResponsesWsEvent,
+    ResponsesApiResponse, streaming::ResponsesStreamEvent,
 };
 use serde_json::{Map, Value};
 
@@ -34,7 +34,7 @@ impl ResponsesWebSocketProviderConfig for OpenAiResponsesApiConfig {
 
     fn transform_ws_request(
         &self,
-        event: &ResponsesWsEvent,
+        event: &ResponsesStreamEvent,
         model: &str,
     ) -> Result<ResponsesWsTransformResult, Error> {
         Ok(ResponsesWsTransformResult::passthrough(enforce_model(
@@ -44,7 +44,7 @@ impl ResponsesWebSocketProviderConfig for OpenAiResponsesApiConfig {
 
     fn transform_ws_response(
         &self,
-        event: &ResponsesWsEvent,
+        event: &ResponsesStreamEvent,
         _model: &str,
     ) -> Result<ResponsesWsTransformResult, Error> {
         Ok(ResponsesWsTransformResult::passthrough(event.clone()))
@@ -230,7 +230,7 @@ mod tests {
 
     #[test]
     fn openai_config_is_native_and_enforces_model() {
-        let event: ResponsesWsEvent =
+        let event: ResponsesStreamEvent =
             serde_json::from_value(serde_json::json!({"type":"response.create"}))
                 .expect("valid event");
         let result = OPENAI_RESPONSES_WS_CONFIG

litellm/__init__.py

@@ -441,7 +441,6 @@ autorouter_presets_url: str = os.getenv(
     "LITELLM_AUTOROUTER_PRESETS_URL",
     "https://raw.githubusercontent.com/BerriAI/litellm/main/litellm/proxy/public_endpoints/autorouter_presets.json",
 )
-whats_new_url: str | None = os.getenv("LITELLM_WHATS_NEW_URL")
 suppress_debug_info: bool = False
 dynamodb_table_name: Optional[str] = None
 s3_callback_params: Optional[Dict] = None

litellm/a2a_protocol/providers/bedrock_agentcore/handler.py

@@ -117,7 +117,7 @@ class BedrockAgentCoreA2AHandler:
         response.raise_for_status()
 
         # Check content type — AgentCore may return JSON instead of SSE
-        content_type: Final = response.headers.get("content-type", "").lower()
+        content_type: Final = ", ".join(response.headers.get_list("content-type")).lower()
 
         if "application/json" in content_type:
             # Single JSON response fallback (not SSE)

litellm/batches/batch_utils.py

@@ -768,7 +768,7 @@ def _get_response_from_batch_job_output_file(
 
 
 def _batch_response_was_successful(
-    batch_job_output_file: Mapping[str, Any], custom_llm_provider: str = "openai"
+    batch_job_output_file: Mapping[str, object], custom_llm_provider: str = "openai"
 ) -> bool:
     """
     Check if the batch job response was successful

litellm/constants.py

@@ -104,6 +104,7 @@ S3_PARTITION_GRANULARITY_ENV_VAR: Final = "S3_PARTITION_GRANULARITY"
 MAX_FILE_LIST_LIMIT: Final = 10000
 DEFAULT_SQS_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_SQS_FLUSH_INTERVAL_SECONDS", 10))
 DEFAULT_NUM_WORKERS_LITELLM_PROXY: Final = int(os.getenv("DEFAULT_NUM_WORKERS_LITELLM_PROXY", 1))
+GRACEFUL_SHUTDOWN_TEARDOWN_HEADROOM_SECONDS: Final = 10
 budget_reservation_disabled_info_emitted = False
 DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE = int(os.getenv("DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE", 1))
 DEFAULT_SQS_BATCH_SIZE: Final = int(os.getenv("DEFAULT_SQS_BATCH_SIZE", 512))
@@ -391,6 +392,11 @@ MODEL_GROUP_ALIAS_RESOLVED_SCOPE_KEY: Final = "litellm.model_group_alias_resolve
 MCP_PEEKED_BODY_SCOPE_KEY: Final = "litellm_mcp_peeked_body"
 MCP_ADMISSION_BODY_PEEK_TIMEOUT_SECONDS: Final = 5.0
 REALTIME_SESSION_SUCCESS_LOGGED_KEY: Final = "realtime_session_success_logged"
+OPENAI_LIVE_SESSION_START_TIMEOUT_SECONDS: Final = 30
+OPENAI_LIVE_SESSION_CLOSE_TIMEOUT_SECONDS: Final = 5
+OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES: Final = frozenset(
+    {"response.completed", "response.incomplete", "response.failed"}
+)
 REALTIME_SESSION_FAILURE_LOGGED_KEY: Final = "realtime_session_failure_logged"
 
 # SSL/TLS cipher configuration for faster handshakes

litellm/cost_calculator.py

@@ -1,14 +1,16 @@
 # What is this?
 ## File for 'response_cost' calculation in Logging
 import logging
+import math
 import time
 from collections.abc import Mapping, Sequence
+from datetime import datetime
 from functools import lru_cache
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, cast
 
 from httpx import Response
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict, ValidationError
 from typing_extensions import ReadOnly, TypedDict, assert_never
 
 import litellm
@@ -17,7 +19,9 @@ from litellm import verbose_logger
 from litellm.constants import (
     DEFAULT_MAX_LRU_CACHE_SIZE,
     DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND,
+    OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES,
 )
+from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.cost_discount import resolve_cost_discount
 from litellm.litellm_core_utils.decisions_pricing import decisions_pricing_fields, is_decisions_call_type
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
@@ -110,7 +114,6 @@ from litellm.types.llms.openai import (
     OpenAIModerationResponse,
     OpenAIRealtimeStreamList,
     OpenAIRealtimeStreamResponseBaseObject,
-    OpenAIRealtimeStreamSessionEvents,
     ResponseAPIUsage,
     ResponsesAPIResponse,
 )
@@ -836,6 +839,22 @@ def _cost_map_entry_prices_anything(entry: Mapping[str, object]) -> bool:
     )
 
 
+def _declares_rates(model_info: ModelInfo | None) -> bool:
+    entry: Final = _raw_cost_map_entry(model_info["key"]) if model_info is not None else None
+    return entry is not None and _cost_map_entry_prices_anything(entry)
+
+
+def _base_model_pricing_provider(base_model: str, custom_llm_provider: str | None) -> str | None:
+    """The provider to price an explicit base_model under, when the deployment's own provider has
+    no rates for it (e.g. chatgpt/ subscription rows) but the base model's own row does"""
+    if custom_llm_provider is None or _declares_rates(_cost_map_model_info(base_model, custom_llm_provider)):
+        return None
+    own_model_info: Final = _cost_map_model_info(base_model, None)
+    if own_model_info is None or not _declares_rates(own_model_info):
+        return None
+    return own_model_info["litellm_provider"]
+
+
 def select_model_name_for_cost_calc(
     model: str | None,
     completion_response: object | None,
@@ -1497,10 +1516,21 @@ def completion_cost(
 
         explicit_pricing: Final = custom_pricing is True or base_model is not None
         custom_priced_request_model: Final = model if custom_pricing is True else None
+        base_model_provider: Final = (
+            _base_model_pricing_provider(
+                base_model,
+                _get_provider_for_cost_calc(
+                    model=model,
+                    custom_llm_provider=custom_llm_provider if isinstance(custom_llm_provider, str) else None,
+                ),
+            )
+            if base_model is not None and custom_pricing is not True
+            else None
+        )
         selected_model: Final = select_model_name_for_cost_calc(
             model=model,
             completion_response=completion_response,
-            custom_llm_provider=custom_llm_provider,
+            custom_llm_provider=base_model_provider or custom_llm_provider,
             custom_pricing=custom_pricing,
             base_model=base_model,
             router_model_id=router_model_id,
@@ -1847,6 +1877,13 @@ def completion_cost(
                 if litellm_logging_obj is not None:
                     request_model_for_cost = litellm_logging_obj.model
 
+                pricing_provider: str | None = (
+                    base_model_provider
+                    if base_model_provider is not None and idx == 0
+                    else custom_llm_provider
+                    if isinstance(custom_llm_provider, str)
+                    else None
+                )
                 (
                     prompt_tokens_cost_usd_dollar,
                     completion_tokens_cost_usd_dollar,
@@ -1854,7 +1891,7 @@ def completion_cost(
                     model=model,
                     prompt_tokens=prompt_tokens or 0,
                     completion_tokens=completion_tokens or 0,
-                    custom_llm_provider=custom_llm_provider,
+                    custom_llm_provider=pricing_provider,
                     response_time_ms=response_time_ms,
                     region_name=None if explicit_pricing else region_name,
                     custom_cost_per_second=custom_cost_per_second,
@@ -1951,12 +1988,9 @@ def completion_cost(
                     _cache_creation_cost: float | None = None
                     _billed_token_rates: BilledTokenRates | None = None
                     if cost_per_token_usage_object is not None and model:
-                        _breakdown_provider: str | None = (
-                            custom_llm_provider if isinstance(custom_llm_provider, str) else None
-                        )
                         _token_type_breakdown = get_token_type_cost_breakdown(
                             model=model,
-                            custom_llm_provider=_breakdown_provider,
+                            custom_llm_provider=pricing_provider,
                             usage=cost_per_token_usage_object,
                             service_tier=service_tier,
                             data_residency=data_residency,
@@ -2979,6 +3013,141 @@ class ResponsesWebSocketTokenUsageProcessor(BaseTokenUsageProcessor):
 _TRANSCRIPTION_COMPLETED_EVENT_TYPE: Final = "conversation.item.input_audio_transcription.completed"
 
 
+class _LiveResponsePayload(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    id: str | None = None
+    model: str | None = None
+    usage: Mapping[str, object] | None = None
+
+
+class _LiveResponseEvent(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: str = ""
+    response: _LiveResponsePayload | None = None
+
+
+class _LiveResponseEnvelope(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    type: str = ""
+    event: _LiveResponseEvent | None = None
+
+
+def _live_response_payload(result: Mapping[str, object]) -> _LiveResponsePayload | None:
+    try:
+        envelope: Final = _LiveResponseEnvelope.model_validate(result)
+    except ValidationError:
+        return None
+    if (
+        envelope.type != "response.event"
+        or envelope.event is None
+        or envelope.event.type not in OPENAI_LIVE_TERMINAL_RESPONSE_EVENT_TYPES
+        or envelope.event.response is None
+        or envelope.event.response.usage is None
+    ):
+        return None
+    return envelope.event.response
+
+
+def _unique_live_response_payloads(results: Sequence[Mapping[str, object]]) -> tuple[_LiveResponsePayload, ...]:
+    response_payloads: Final = tuple(
+        response for result in results if (response := _live_response_payload(result)) is not None
+    )
+    first_payload_by_id: Final = {
+        response.id: response for response in reversed(response_payloads) if response.id is not None
+    }
+    return tuple(reversed(first_payload_by_id.values()))
+
+
+def _live_session_model(result: Mapping[str, object]) -> str | None:
+    session: Final = as_str_mapping(result.get("session"))
+    if session is None:
+        return None
+    model: Final = session.get("model")
+    return model if isinstance(model, str) else None
+
+
+def _live_usage_seconds(result: Mapping[str, object]) -> float | None:
+    usage: Final = as_str_mapping(result.get("usage"))
+    if usage is None:
+        return None
+    seconds: Final = usage.get("seconds")
+    if isinstance(seconds, (int, float)) and not isinstance(seconds, bool) and math.isfinite(seconds):
+        return float(seconds)
+    return None
+
+
+def _last_live_usage_seconds(results: Sequence[Mapping[str, object]], event_type: str) -> float | None:
+    return next(
+        (
+            seconds
+            for result in reversed(results)
+            if result.get("type") == event_type and (seconds := _live_usage_seconds(result)) is not None
+        ),
+        None,
+    )
+
+
+def _live_session_wall_clock_seconds(litellm_logging_obj: LitellmLoggingObject | None) -> float:
+    if litellm_logging_obj is None:
+        return 0.0
+    model_call_details: Final = litellm_logging_obj.model_call_details
+    start_time: Final = model_call_details.get("start_time")
+    end_time: Final = model_call_details.get("end_time")
+    if isinstance(start_time, datetime) and isinstance(end_time, datetime):
+        return max((end_time - start_time).total_seconds(), 0.0)
+    return 0.0
+
+
+def _live_input_cost_per_second(model_name: str, custom_llm_provider: str) -> float | None:
+    model_info: Final = _lookup_model_info_or_none(model_name, custom_llm_provider)
+    if model_info is None:
+        return None
+    rate: Final = model_info.get("input_cost_per_second")
+    if isinstance(rate, (int, float)) and not isinstance(rate, bool) and math.isfinite(rate):
+        return float(rate)
+    return None
+
+
+def _first_live_input_cost_per_second(
+    potential_model_names: Sequence[str | None],
+    custom_llm_provider: str,
+) -> float:
+    return next(
+        (
+            rate
+            for model_name in potential_model_names
+            if model_name is not None
+            and (rate := _live_input_cost_per_second(model_name, custom_llm_provider)) is not None
+        ),
+        0.0,
+    )
+
+
+def _live_response_token_costs(
+    response: _LiveResponsePayload,
+    custom_llm_provider: str,
+    data_residency: str | None,
+) -> tuple[float, float]:
+    if response.id is None or response.model is None or response.usage is None:
+        return 0.0, 0.0
+    try:
+        usage: Final = get_usage_object(response.model_dump())
+    except (ValidationError, TypeError, ValueError, KeyError, AttributeError):
+        return 0.0, 0.0
+    if usage is None:
+        return 0.0, 0.0
+    costs: Final = _candidate_realtime_token_costs(
+        model_name=response.model,
+        combined_usage_object=usage,
+        custom_llm_provider=custom_llm_provider,
+        data_residency=data_residency,
+    )
+    return (0.0, 0.0) if costs is None else costs
+
+
 def _candidate_realtime_token_costs(
     model_name: str,
     combined_usage_object: Usage,
@@ -3065,14 +3234,21 @@ def handle_realtime_stream_cost_calculation(
         base_pricing_model: the deployment's resolved base_model, tried ahead of the
             session-reported model but after custom rates
     """
-    received_model = None
-    potential_model_names: Final = [custom_pricing_model, base_pricing_model]
-    for result in results:
-        if result["type"] == "session.created":
-            received_model = cast(OpenAIRealtimeStreamSessionEvents, result)["session"].get("model", None)
-            potential_model_names.append(received_model)
-
-    potential_model_names.append(litellm_model_name)
+    received_model: Final = next(
+        (
+            model_name
+            for result in results
+            if result["type"] in ("session.created", "session.started")
+            and (model_name := _live_session_model(result)) is not None
+        ),
+        None,
+    )
+    potential_model_names: Final = (
+        custom_pricing_model,
+        base_pricing_model,
+        received_model,
+        litellm_model_name,
+    )
     input_cost_per_token, output_cost_per_token = _first_priced_realtime_token_costs(
         potential_model_names=potential_model_names,
         combined_usage_object=combined_usage_object,
@@ -3089,15 +3265,56 @@ def handle_realtime_stream_cost_calculation(
         if any(r.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE for r in results)
         else 0.0
     )
-    total_cost: Final = input_cost_per_token + output_cost_per_token + transcription_cost
+    live_lifecycle_event_types: Final = frozenset({"session.started", "session.usage.updated", "session.closed"})
+    live_session_started: Final = any(result["type"] == "session.started" for result in results)
+    live_session_closed: Final = any(result["type"] == "session.closed" for result in results)
+    live_event_seen: Final = any(result["type"] in live_lifecycle_event_types for result in results)
+    closed_seconds: Final = _last_live_usage_seconds(results, "session.closed")
+    reported_seconds: Final = _last_live_usage_seconds(results, "session.usage.updated")
+    latest_reported_seconds: Final = 0.0 if reported_seconds is None else reported_seconds
+    voice_seconds: Final = (
+        closed_seconds
+        if closed_seconds is not None
+        else max(
+            latest_reported_seconds,
+            _live_session_wall_clock_seconds(litellm_logging_obj),
+        )
+        if live_session_started and not live_session_closed
+        else latest_reported_seconds
+    )
+    live_model_names: Final = (
+        (custom_pricing_model, base_pricing_model, received_model, litellm_model_name) if live_event_seen else ()
+    )
+    live_voice_cost: Final = voice_seconds * _first_live_input_cost_per_second(
+        live_model_names,
+        custom_llm_provider,
+    )
+    live_response_costs: Final = tuple(
+        _live_response_token_costs(
+            response=response,
+            custom_llm_provider=custom_llm_provider,
+            data_residency=data_residency,
+        )
+        for response in _unique_live_response_payloads(results)
+    )
+    live_response_input_cost: Final = sum(costs[0] for costs in live_response_costs)
+    live_response_output_cost: Final = sum(costs[1] for costs in live_response_costs)
+    input_cost: Final = input_cost_per_token + live_response_input_cost
+    output_cost: Final = output_cost_per_token + live_response_output_cost
+    total_cost: Final = input_cost + output_cost + transcription_cost + live_voice_cost
+    additional_costs: Final = tuple(
+        (cost_name, cost)
+        for cost_name, cost in (("transcription_cost", transcription_cost), ("live_voice_cost", live_voice_cost))
+        if cost > 0
+    )
 
     _store_cost_breakdown_in_logging_obj(
         litellm_logging_obj=litellm_logging_obj,
-        prompt_tokens_cost_usd_dollar=input_cost_per_token,
-        completion_tokens_cost_usd_dollar=output_cost_per_token,
+        prompt_tokens_cost_usd_dollar=input_cost,
+        completion_tokens_cost_usd_dollar=output_cost,
         cost_for_built_in_tools_cost_usd_dollar=0.0,
         total_cost_usd_dollar=total_cost,
-        additional_costs={"transcription_cost": transcription_cost} if transcription_cost > 0 else None,
+        additional_costs=dict(additional_costs) if additional_costs else None,
         data_residency=data_residency,
     )
 

litellm/harness/handlers/deepagents_handler.py

@@ -158,7 +158,7 @@ class DeepAgentsHandler(BaseHarnessHandler):
         agent, deps = self._require_agent()
         snapshot = await agent.aget_state(self._run_config(ctx, None))
         messages = (snapshot.values or MappingProxyType({})).get("messages") or ()
-        converted: list[dict[str, Any]] = deps.convert_to_openai_messages(  # mutable-ok: LangChain returns a list
+        converted: list[dict[str, object]] = deps.convert_to_openai_messages(  # mutable-ok: LangChain returns a list
             messages
         )
         return converted

litellm/integrations/anthropic_cache_control_hook.py

@@ -23,6 +23,7 @@ from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.custom_prompt_management import CustomPromptManagement
 from litellm.integrations.prompt_management_base import PromptManagementClient
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    cache_control_as_prompt_cache_breakpoint,
     with_prompt_cache_breakpoint,
 )
 from litellm.llms.anthropic.common_utils import (
@@ -72,6 +73,7 @@ OPENAI_API_HOST: Final = "api.openai.com"
 OPENAI_API_BASE_ENV_VARS: Final = ("OPENAI_BASE_URL", "OPENAI_API_BASE")
 _OBJECT_MAPPING_ADAPTER: Final = TypeAdapter(dict[object, object])
 _OBJECT_LIST_ADAPTER: Final = TypeAdapter(list[object])
+_CONTENT_PART_ADAPTER: Final = TypeAdapter(dict[str, object])
 
 AllToolParamValues = ChatCompletionToolParam | AllAnthropicToolsValues
 
@@ -90,6 +92,12 @@ def _validated_object_list(value: object) -> list[object] | None:
         return None
 
 
+def _part_with_cache_control_renamed(part: object) -> object:
+    if not _accepts_prompt_cache_breakpoint(part):
+        return part
+    return cache_control_as_prompt_cache_breakpoint(_CONTENT_PART_ADAPTER.validate_python(part))
+
+
 def configured_injection_points(value: object) -> Sequence[CacheControlInjectionPoint]:
     if not isinstance(value, (list, tuple)):
         return ()
@@ -242,6 +250,10 @@ def _accepts_prompt_cache_breakpoint(block: object) -> bool:
     return isinstance(block, dict) and block.get("type") in OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES
 
 
+def _may_carry_prompt_cache_breakpoint(message: Mapping[str, object]) -> bool:
+    return message.get("role") != "assistant"
+
+
 # Set by a caller whose message list is not the one that goes upstream -- today the
 # Responses API layer, whose `instructions` only becomes a system message further down.
 # Tells this hook to hand role-targeted points to the pass holding the final messages
@@ -620,9 +632,32 @@ class AnthropicCacheControlHook(CustomPromptManagement):
                 message_content[-1]["cache_control"] = control  # pyright: ignore[reportGeneralTypeIssues]  # loose runtime dict
         return message
 
+    @staticmethod
+    def cache_control_as_prompt_cache_breakpoints(message: AllMessageValues) -> AllMessageValues:
+        """Every Anthropic ``cache_control`` on the message or its parts moved to OpenAI's ``prompt_cache_breakpoint``,
+        placed where the injection hook would put one. Marks with no OpenAI slot are left for the caller to strip."""
+        fields: Final[Mapping[str, object]] = dict(message)
+        if not _may_carry_prompt_cache_breakpoint(fields):
+            return message
+        parts: Final = _validated_object_list(fields.get("content"))
+        renamed: Final = (
+            message
+            if parts is None
+            else cast(  # cast-ok: the same message with its part marks renamed
+                AllMessageValues,
+                {
+                    **fields,
+                    "content": [_part_with_cache_control_renamed(part) for part in parts],
+                },
+            )
+        )
+        if fields.get("cache_control") is None:
+            return renamed
+        return AnthropicCacheControlHook._insert_prompt_cache_breakpoint_in_message(renamed)
+
     @staticmethod
     def _insert_prompt_cache_breakpoint_in_message(message: AllMessageValues) -> AllMessageValues:
-        if message.get("role") == "assistant":
+        if not _may_carry_prompt_cache_breakpoint(message):
             return message
         message_content: Final = message.get("content", None)
         if isinstance(message_content, str):

litellm/integrations/argilla.py

@@ -50,7 +50,7 @@ class ArgillaLogger(CustomBatchLogger):
         if litellm.argilla_transformation_object is None:
             raise Exception("'litellm.argilla_transformation_object' is required, to log your payload to Argilla.")
         self.validate_argilla_transformation_object(litellm.argilla_transformation_object)
-        self.argilla_transformation_object = litellm.argilla_transformation_object
+        self.argilla_transformation_object: Mapping[str, str] = litellm.argilla_transformation_object
         self.default_credentials = self.get_credentials_from_env(
             argilla_api_key=argilla_api_key,
             argilla_dataset_name=argilla_dataset_name,

litellm/integrations/arize/arize_phoenix_client.py

@@ -3,7 +3,7 @@ Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
 """
 
 import urllib.parse
-from typing import Any, Final
+from typing import Final
 
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
@@ -53,7 +53,7 @@ class ArizePhoenixClient:
         # Initialize HTTPHandler
         self.http_handler = HTTPHandler(disable_default_headers=True)
 
-    def get_prompt_version(self, prompt_version_id: str) -> dict[str, Any] | None:
+    def get_prompt_version(self, prompt_version_id: str) -> dict[str, object] | None:
         """
         Fetch a prompt version from Arize Phoenix.
 

litellm/integrations/clickhouse/clickhouse_spend_logger.py

@@ -80,7 +80,7 @@ def _json(value: object) -> str:
     return value if isinstance(value, str) else json.dumps(value, default=str)
 
 
-def _json_mapping(value: Mapping[str, Any]) -> str:
+def _json_mapping(value: Mapping[str, object]) -> str:
     return _json(dict(value))
 
 

litellm/integrations/cloudzero/cz_stream_api.py

@@ -209,7 +209,7 @@ class CloudZeroStreamer:
 
         return payload
 
-    def _convert_cbf_to_api_format(self, row: dict[str, object]) -> dict[str, Any] | None:
+    def _convert_cbf_to_api_format(self, row: dict[str, object]) -> dict[str, object] | None:
         """Convert CBF row to CloudZero API format - keeping CBF field names as CloudZero expects them."""
         try:
             # CloudZero expects CBF format field names directly, not converted names

litellm/integrations/custom_guardrail.py

@@ -511,7 +511,7 @@ class CustomGuardrail(CustomLogger):
             sticky_session_routing=self.sticky_session_routing,
         )
 
-    def _get_session_id_from_request_data(self, request_data: dict[str, Any]) -> str | None:
+    def _get_session_id_from_request_data(self, request_data: dict[str, object]) -> str | None:
         """Extract session_id from request data."""
         return get_session_id_from_request_data(request_data)
 

litellm/integrations/dotprompt/prompt_manager.py

@@ -70,7 +70,7 @@ class PromptManager:
         self,
         prompt_id: str | None = None,
         prompt_directory: str | None = None,
-        prompt_data: dict[str, dict[str, Any]] | None = None,
+        prompt_data: dict[str, dict[str, object]] | None = None,
         prompt_file: str | None = None,
     ):
         self.prompt_directory = Path(prompt_directory) if prompt_directory else None
@@ -372,6 +372,6 @@ class PromptManager:
             }
         return result
 
-    def load_prompts_from_json_data(self, prompt_data: dict[str, dict[str, Any]]) -> None:
+    def load_prompts_from_json_data(self, prompt_data: dict[str, dict[str, object]]) -> None:
         """Load additional prompts from JSON data (merges with existing prompts)."""
         self._load_prompts_from_json(prompt_data)

litellm/integrations/focus/focus_logger.py

@@ -115,7 +115,7 @@ class FocusLogger(CustomLogger):
 
         pod_lock_manager: PodLockManager | None = None
         if proxy_logging_obj is not None:
-            writer: Final = getattr(proxy_logging_obj, "db_spend_update_writer", None)
+            writer: Final[object] = getattr(proxy_logging_obj, "db_spend_update_writer", None)
             if writer is not None:
                 pod_lock_manager = getattr(writer, "pod_lock_manager", None)
 

litellm/integrations/otel/model/spans.py

@@ -335,6 +335,7 @@ _PRISMA_MODELS: Final[frozenset[str]] = frozenset(
         "LiteLLM_PolicyTable",
         "LiteLLM_PolicyAttachmentTable",
         "LiteLLM_ToolTable",
+        "LiteLLM_TelemetryReport",
         "LiteLLM_AccessGroupTable",
         "LiteLLM_ClaudeCodePluginTable",
         "LiteLLM_MemoryTable",

litellm/integrations/vantage/vantage_logger.py

@@ -16,6 +16,8 @@ from litellm.integrations.focus.focus_logger import FocusLogger
 
 if TYPE_CHECKING:
     from apscheduler.schedulers.asyncio import AsyncIOScheduler
+
+    from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
 else:
     AsyncIOScheduler = Any
 
@@ -91,7 +93,7 @@ class VantageLogger(FocusLogger):
         """
         from litellm.proxy.proxy_server import proxy_logging_obj
 
-        pod_lock_manager = None
+        pod_lock_manager: PodLockManager | None = None
         if proxy_logging_obj is not None:
             writer: Final[object] = getattr(proxy_logging_obj, "db_spend_update_writer", None)
             if writer is not None:

litellm/litellm_core_utils/core_helpers.py

@@ -52,6 +52,10 @@ def safe_divide_seconds(seconds: float, denominator: float, default: float | Non
 _DROP_PARAMS_BOOL: Final = TypeAdapter(bool)
 
 
+def as_str_mapping(value: object) -> Mapping[str, object] | None:
+    return value if isinstance(value, Mapping) else None
+
+
 def normalize_drop_params(value: object) -> bool | None:
     if value is None or isinstance(value, bool):
         return value

litellm/litellm_core_utils/get_blog_posts.py

@@ -8,7 +8,6 @@ Disable remote fetching entirely:
     export LITELLM_LOCAL_BLOG_POSTS=True
 """
 
-import json
 import os
 import time
 import xml.etree.ElementTree as ET
@@ -17,23 +16,53 @@ from importlib.resources import files
 from typing import Final
 
 import httpx
+from pydantic import ConfigDict
 
 from litellm import verbose_logger
 from litellm.types.llms.base import LiteLLMBaseModel
 
 BLOG_POSTS_TTL_SECONDS: Final[int] = 3600  # 1 hour
-BLOG_POSTS_MAX: Final[int] = 6
+BLOG_POSTS_MAX: Final[int] = 50
 
 
 class BlogPost(LiteLLMBaseModel):
+    model_config = ConfigDict(frozen=True)
+
     title: str
     description: str
     date: str
     url: str
+    tags: tuple[str, ...] = ()
 
 
 class BlogPostsResponse(LiteLLMBaseModel):
-    posts: list[BlogPost]
+    model_config = ConfigDict(frozen=True)
+
+    posts: tuple[BlogPost, ...]
+
+
+def _pub_date(pub_date_el: ET.Element | None) -> str:
+    if pub_date_el is None or not pub_date_el.text:
+        return ""
+    try:
+        return parsedate_to_datetime(pub_date_el.text).strftime("%Y-%m-%d")
+    except Exception:
+        return pub_date_el.text
+
+
+def _parse_item(item: ET.Element) -> BlogPost | None:
+    title_el: Final = item.find("title")
+    link_el: Final = item.find("link")
+    if title_el is None or link_el is None:
+        return None
+    desc_el: Final = item.find("description")
+    return BlogPost(
+        title=title_el.text or "",
+        description=(desc_el.text or "") if desc_el is not None else "",
+        date=_pub_date(item.find("pubDate")),
+        url=link_el.text or "",
+        tags=tuple(category.text for category in item.findall("category") if category.text),
+    )
 
 
 class GetBlogPosts:
@@ -41,19 +70,20 @@ class GetBlogPosts:
     Fetches, validates, and caches LiteLLM blog posts.
 
     - Fetches RSS feed from docs site with a 5-second timeout
-    - Parses the XML and extracts the latest BLOG_POSTS_MAX blog posts
+    - Parses the XML and extracts up to BLOG_POSTS_MAX blog posts, with their RSS categories as tags
     - Caches the result in-process for BLOG_POSTS_TTL_SECONDS (1 hour)
     - Falls back to the bundled local backup on any failure
     """
 
-    _cached_posts: list[dict[str, str]] | None = None
+    _cached_posts: tuple[BlogPost, ...] | None = None
     _last_fetch_time: float = 0.0
 
     @staticmethod
-    def load_local_blog_posts() -> list[dict[str, str]]:
+    def load_local_blog_posts() -> tuple[BlogPost, ...]:
         """Load the bundled local backup blog posts."""
-        content: Final = json.loads(files("litellm").joinpath("blog_posts.json").read_text(encoding="utf-8"))
-        return content.get("posts", [])
+        return BlogPostsResponse.model_validate_json(
+            files("litellm").joinpath("blog_posts.json").read_text(encoding="utf-8")
+        ).posts
 
     @staticmethod
     def fetch_rss_feed(url: str, timeout: int = 5) -> str:
@@ -67,54 +97,24 @@ class GetBlogPosts:
         return response.text
 
     @staticmethod
-    def parse_rss_to_posts(xml_text: str, max_posts: int = BLOG_POSTS_MAX) -> list[dict[str, str]]:
+    def parse_rss_to_posts(xml_text: str, max_posts: int = BLOG_POSTS_MAX) -> tuple[BlogPost, ...]:
         """
-        Parse RSS XML and return a list of blog post dicts.
+        Parse RSS XML into blog posts.
 
-        Extracts title, description, date (YYYY-MM-DD), and url from each <item>.
+        Extracts title, description, date (YYYY-MM-DD), url and the <category> tags from each <item>.
         """
         root: Final = ET.fromstring(xml_text)
         channel: Final = root.find("channel")
         if channel is None:
             raise ValueError("RSS feed missing <channel> element")
 
-        posts: Final[list[dict[str, str]]] = []
-        for item in channel.findall("item"):
-            if len(posts) >= max_posts:
-                break
-
-            title_el = item.find("title")
-            link_el = item.find("link")
-            desc_el = item.find("description")
-            pub_date_el = item.find("pubDate")
-
-            if title_el is None or link_el is None:
-                continue
-
-            # Parse RFC 2822 date to YYYY-MM-DD
-            date_str = ""
-            if pub_date_el is not None and pub_date_el.text:
-                try:
-                    dt = parsedate_to_datetime(pub_date_el.text)
-                    date_str = dt.strftime("%Y-%m-%d")
-                except Exception:
-                    date_str = pub_date_el.text
-
-            posts.append(
-                {
-                    "title": title_el.text or "",
-                    "description": desc_el.text or "" if desc_el is not None else "",
-                    "date": date_str,
-                    "url": link_el.text or "",
-                }
-            )
-
-        return posts
+        parsed: Final = (_parse_item(item) for item in channel.findall("item"))
+        return tuple(post for post in parsed if post is not None)[:max_posts]
 
     @staticmethod
-    def validate_blog_posts(posts: list[dict[str, str]]) -> bool:
-        """Return True if posts is a non-empty list."""
-        if not isinstance(posts, list) or len(posts) == 0:
+    def validate_blog_posts(posts: tuple[BlogPost, ...]) -> bool:
+        """Return True if there is at least one post."""
+        if len(posts) == 0:
             verbose_logger.warning(
                 "LiteLLM: Parsed RSS feed has no valid posts. Falling back to local backup.",
             )
@@ -122,9 +122,9 @@ class GetBlogPosts:
         return True
 
     @classmethod
-    def get_blog_posts(cls, url: str) -> list[dict[str, str]]:
+    def get_blog_posts(cls, url: str) -> tuple[BlogPost, ...]:
         """
-        Return the blog posts list.
+        Return the blog posts.
 
         Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS.
         Fetches from ``url`` otherwise, falling back to local backup on failure.
@@ -156,6 +156,6 @@ class GetBlogPosts:
         return posts
 
 
-def get_blog_posts(url: str) -> list[dict[str, str]]:
-    """Public entry point — returns the blog posts list."""
+def get_blog_posts(url: str) -> tuple[BlogPost, ...]:
+    """Public entry point, returns the blog posts."""
     return GetBlogPosts.get_blog_posts(url=url)

litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py

@@ -709,7 +709,7 @@ def convert_to_model_response_object(
 
                     message = Message(
                         content=content,
-                        role=choice["message"]["role"] or "assistant",
+                        role=choice["message"].get("role") or "assistant",
                         function_call=choice["message"].get("function_call", None),
                         tool_calls=tool_calls,
                         audio=choice["message"].get("audio", None),
@@ -720,6 +720,8 @@ def convert_to_model_response_object(
                         images=_normalize_images_for_message(choice["message"].get("images", None)),
                     )
                     finish_reason = choice.get("finish_reason", None)
+                if "refusal" in choice["message"]:
+                    message.refusal = choice["message"]["refusal"]
                 if finish_reason is None:
                     # gpt-4 vision can return 'finish_reason' or 'finish_details'
                     finish_reason = choice.get("finish_details") or "stop"
@@ -728,9 +730,10 @@ def convert_to_model_response_object(
 
                 ## PROVIDER SPECIFIC FIELDS ##
                 provider_specific_fields = {f: choice[f] for f in choice.keys() - _CHOICES_FIELDS}
-
                 logprobs = choice.get("logprobs", None)
+                provider_sent_logprobs = "logprobs" in choice
                 enhancements = choice.get("enhancements", None)
+
                 choice = Choices(
                     finish_reason=finish_reason,
                     index=idx,
@@ -739,6 +742,9 @@ def convert_to_model_response_object(
                     enhancements=enhancements,
                     provider_specific_fields=provider_specific_fields,
                 )
+                if provider_sent_logprobs and logprobs is None:
+                    choice.logprobs = None
+
                 choice_list.append(choice)
             model_response_object.choices = choice_list
 

litellm/litellm_core_utils/logging_worker.py

@@ -466,7 +466,7 @@ class LoggingWorker:
         # Process all tasks concurrently for maximum speed
         await asyncio.gather(*[self._process_single_task(task) for task in tasks])
 
-    def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine):
+    def ensure_initialized_and_enqueue(self, async_coroutine: Coroutine[object, None, None]) -> None:
         """
         Ensure the logging worker is initialized and enqueue the coroutine.
         """

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -1788,7 +1788,8 @@ _MarkedT: Final = TypeVar("_MarkedT", bound=Mapping[str, object])
 
 
 def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
-    if marker is None:
+    """A breakpoint the client already wrote on the block wins, TTL included."""
+    if marker is None or target.get("prompt_cache_breakpoint") is not None:
         return target
     marked: Final = {**target, "prompt_cache_breakpoint": marker}
     return cast(_MarkedT, marked)  # cast-ok: same block shape as the input plus the marker key

litellm/litellm_core_utils/redact_messages.py

@@ -108,6 +108,7 @@ def _redact_choice_content(choice):
             choice.message.reasoning_content = REDACTED_BY_LITELLM
         if hasattr(choice.message, "thinking_blocks"):
             choice.message.thinking_blocks = None
+        _redact_provider_specific_fields(choice.message)
         _redact_tool_calls(getattr(choice.message, "tool_calls", None))
         _redact_function_call(getattr(choice.message, "function_call", None))
     elif isinstance(choice, litellm.utils.StreamingChoices):
@@ -117,10 +118,39 @@ def _redact_choice_content(choice):
             choice.delta.reasoning_content = REDACTED_BY_LITELLM
         if hasattr(choice.delta, "thinking_blocks"):
             choice.delta.thinking_blocks = None
+        _redact_provider_specific_fields(choice.delta)
         _redact_tool_calls(getattr(choice.delta, "tool_calls", None))
         _redact_function_call(getattr(choice.delta, "function_call", None))
 
 
+def _as_mapping(value: object) -> Mapping[str, object] | None:
+    return value if isinstance(value, Mapping) else None  # pyright: ignore[reportUnknownVariableType]  # keys are read one by one
+
+
+def _redact_provider_specific_fields(message: object) -> None:
+    if (fields := _as_mapping(getattr(message, "provider_specific_fields", None))) is not None:
+        setattr(message, "provider_specific_fields", _redacted_provider_specific_fields(fields))  # noqa: B010  # typed as object
+
+
+def _redact_provider_specific_fields_dict(choice: Mapping[str, object]) -> None:
+    for message in (choice.get("message"), choice.get("delta")):
+        if (
+            (readable := _as_mapping(message)) is not None
+            and (fields := _as_mapping(readable.get("provider_specific_fields"))) is not None
+            and isinstance(message, dict)
+        ):
+            message["provider_specific_fields"] = _redacted_provider_specific_fields(fields)
+
+
+def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[str, object]:
+    """Copy of provider_specific_fields with its reasoning redacted the same way as the message's own."""
+    return {
+        **fields,
+        **({"reasoning_content": REDACTED_BY_LITELLM} if fields.get("reasoning_content") is not None else {}),
+        **{key: None for key in ("thinking_blocks", "reasoningContentBlocks", "citations") if key in fields},
+    }
+
+
 def _redact_responses_api_output(output_items):
     """Helper to redact ResponsesAPIResponse output items."""
     for output_item in output_items:
@@ -228,6 +258,7 @@ def _redact_tool_calls_dict(message: Mapping[str, object]) -> None:
 def _redact_model_response_dict_choices(choices, redacted_str: str):
     for choice in choices:
         if isinstance(choice, dict):
+            _redact_provider_specific_fields_dict(choice)
             if "message" in choice and isinstance(choice["message"], dict):
                 if choice["message"].get("content") is not None:
                     choice["message"]["content"] = redacted_str
@@ -261,8 +292,37 @@ def _redacted_baseline_metadata(metadata: Mapping[str, object]) -> Mapping[str,
     return {**metadata, "_autorouter_baseline_route": replace(route, request_parameters=None)}
 
 
-def redacted_litellm_params(params: Mapping[str, object]) -> dict[str, object]:
-    request: Final = params.get("proxy_server_request")
+def _redacted_proxy_server_request(request: Mapping[str, object], *, redact_body: bool) -> Mapping[str, object]:
+    """Copy of proxy_server_request; redact_body redacts its body the same way as the top-level request."""
+    body: Final = _as_mapping(request.get("body"))
+    if not redact_body or body is None:
+        return without_classifier_audit(request)
+    redacted_body: Final = {**body}
+    perform_redaction(redacted_body, result=None)
+    return {**without_classifier_audit(request), "body": redacted_body}
+
+
+def _redact_hidden_provider_specific_fields(params: Mapping[str, object]) -> None:
+    """Redact the provider_specific_fields copied into metadata hidden_params, keeping each metadata dict."""
+    for metadata in (params.get("metadata"), params.get("litellm_metadata")):
+        if (
+            (readable := _as_mapping(metadata)) is not None
+            and (hidden_params := _as_mapping(readable.get("hidden_params"))) is not None
+            and (fields := _as_mapping(hidden_params.get("provider_specific_fields"))) is not None
+            and isinstance(metadata, dict)
+        ):
+            metadata["hidden_params"] = {
+                **hidden_params,
+                "provider_specific_fields": _redacted_provider_specific_fields(fields),
+            }
+
+
+def redacted_litellm_params(params: Mapping[str, object], *, redact_request_body: bool = True) -> dict[str, object]:
+    """
+    Copy of litellm_params without classifier audit fields or baseline route request parameters.
+    redact_request_body also redacts the nested proxy_server_request body.
+    """
+    request: Final = _as_mapping(params.get("proxy_server_request"))
     return {
         **params,
         **{
@@ -270,7 +330,11 @@ def redacted_litellm_params(params: Mapping[str, object]) -> dict[str, object]:
             for key, value in params.items()
             if key in ("metadata", "litellm_metadata") and isinstance(value, Mapping)
         },
-        **({"proxy_server_request": without_classifier_audit(request)} if isinstance(request, Mapping) else {}),
+        **(
+            {"proxy_server_request": _redacted_proxy_server_request(request, redact_body=redact_request_body)}
+            if request is not None
+            else {}
+        ),
     }
 
 
@@ -280,12 +344,17 @@ def perform_redaction(model_call_details: dict, result, redact_streaming_respons
 
     redact_streaming_responses=False skips the in-place redaction of the shared streaming
     response entries; per-callback redaction hands each opted-out callback its own redacted
-    copy via redact_streaming_responses_for_custom_logger instead.
+    copy via redact_streaming_responses_for_custom_logger instead. It also leaves the shared
+    proxy request body and hidden provider fields as they are, since the spend log reads them.
     """
     # Redact model_call_details
-    params: Final = model_call_details.get("litellm_params")
-    if isinstance(params, Mapping):
-        model_call_details["litellm_params"] = redacted_litellm_params(params)
+    params: Final = _as_mapping(model_call_details.get("litellm_params"))
+    if params is not None:
+        if redact_streaming_responses:
+            _redact_hidden_provider_specific_fields(params)
+        model_call_details["litellm_params"] = redacted_litellm_params(
+            params, redact_request_body=redact_streaming_responses
+        )
     model_call_details["messages"] = [{"role": "user", "content": REDACTED_BY_LITELLM}]
     model_call_details["prompt"] = ""
     model_call_details["input"] = ""

litellm/llms/a2a/chat/transformation.py

@@ -7,6 +7,7 @@ from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import TYPE_CHECKING, Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.llms.azure_ai.common_utils import AZURE_ENTRA_LITELLM_PARAM_KEYS, get_azure_ai_agent_entra_token
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
@@ -30,6 +31,7 @@ if TYPE_CHECKING:
 _REGISTRY_PARAMS_KEPT_OUT_OF_OPTIONAL_PARAMS: Final = (
     frozenset({"api_key", "api_base", "headers", "model"}) | AZURE_ENTRA_LITELLM_PARAM_KEYS
 )
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
 
 
 def _card_declares_no_streaming(agent_card_params: Mapping[str, object]) -> bool:
@@ -325,7 +327,7 @@ class A2AConfig(BaseConfig):
 
         # Check for JSON-RPC error
         if "error" in response_json:
-            error: Final = response_json["error"]
+            error: Final = _JSON_OBJECT.validate_python(response_json["error"])
             raise A2AError(
                 status_code=raw_response.status_code,
                 message=f"A2A error: {error.get('message', 'Unknown error')}",
@@ -333,7 +335,7 @@ class A2AConfig(BaseConfig):
             )
 
         # Extract text from A2A response
-        text: Final = extract_text_from_a2a_response(response_json)
+        text: Final = extract_text_from_a2a_response(_JSON_OBJECT.validate_python(response_json))
 
         # Populate model response
         model_response.choices = [

litellm/llms/a2a/common_utils.py

@@ -22,7 +22,7 @@ class A2AError(BaseLLMException):
         self,
         status_code: int,
         message: str,
-        headers: dict[str, Any] = {},
+        headers: dict[str, str] = {},
     ):
         super().__init__(
             status_code=status_code,

litellm/llms/aiml/image_generation/transformation.py

@@ -1,6 +1,8 @@
+from collections.abc import Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.llms.base_llm.image_generation.transformation import (
     BaseImageGenerationConfig,
@@ -21,6 +23,10 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObj = Any
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_CONTAINER: Final[TypeAdapter[Mapping[str, object] | str | Sequence[object]]] = TypeAdapter(
+    Mapping[str, object] | str | Sequence[object], config=ConfigDict(hide_input_in_errors=True)
+)
 
 OPENAI_STYLE_IMAGE_MODEL_PREFIXES: Final[tuple[str, ...]] = ("openai/",)
 
@@ -198,29 +204,33 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig):
 
         if "data" in response_data and isinstance(response_data["data"], list):
             # Handle OpenAI-like format: {"data": [{"url": "...", "width": 1024, "height": 768, "content_type": "image/jpeg"}]}
-            for image in response_data["data"]:
+            for raw_image in _JSON_CONTAINER.validate_python(response_data["data"]):
+                image = raw_image if isinstance(raw_image, str) else _JSON_CONTAINER.validate_python(raw_image)
                 if "url" in image:
+                    url_image = _JSON_OBJECT.validate_python(image)
                     model_response.data.append(
                         ImageObject(
                             b64_json=None,
-                            url=image["url"],
-                            revised_prompt=image.get("revised_prompt"),
+                            url=url_image["url"],
+                            revised_prompt=url_image.get("revised_prompt"),
                         )
                     )
                 elif "b64_json" in image or "image_base64" in image:
+                    b64_image = _JSON_OBJECT.validate_python(image)
                     model_response.data.append(
                         ImageObject(
-                            b64_json=image.get("b64_json") or image.get("image_base64"),
+                            b64_json=b64_image.get("b64_json") or b64_image.get("image_base64"),
                             url=None,
-                            revised_prompt=image.get("revised_prompt"),
+                            revised_prompt=b64_image.get("revised_prompt"),
                         )
                     )
         elif "output" in response_data and "choices" in response_data["output"]:
-            for choice in response_data["output"]["choices"]:
+            for raw_choice in _JSON_CONTAINER.validate_python(response_data["output"]["choices"]):
+                choice = raw_choice if isinstance(raw_choice, str) else _JSON_CONTAINER.validate_python(raw_choice)
                 if "image_base64" in choice:
                     model_response.data.append(
                         ImageObject(
-                            b64_json=choice["image_base64"],
+                            b64_json=_JSON_OBJECT.validate_python(choice)["image_base64"],
                             url=None,
                         )
                     )
@@ -228,23 +238,24 @@ class AimlImageGenerationConfig(BaseImageGenerationConfig):
                     model_response.data.append(
                         ImageObject(
                             b64_json=None,
-                            url=choice["url"],
+                            url=_JSON_OBJECT.validate_python(choice)["url"],
                         )
                     )
         elif "images" in response_data:
             # Handle alternative format: {"images": [{"url": "...", "width": 1024, "height": 768, "content_type": "image/jpeg"}]}
-            for image in response_data["images"]:
+            for raw_image in _JSON_CONTAINER.validate_python(response_data["images"]):
+                image = raw_image if isinstance(raw_image, str) else _JSON_CONTAINER.validate_python(raw_image)
                 if "url" in image:
                     model_response.data.append(
                         ImageObject(
                             b64_json=None,
-                            url=image["url"],
+                            url=_JSON_OBJECT.validate_python(image)["url"],
                         )
                     )
                 elif "image_base64" in image:
                     model_response.data.append(
                         ImageObject(
-                            b64_json=image["image_base64"],
+                            b64_json=_JSON_OBJECT.validate_python(image)["image_base64"],
                             url=None,
                         )
                     )

litellm/llms/anthropic/chat/handler.py

@@ -3,7 +3,9 @@ Calling + translation logic for anthropic's `/v1/messages` endpoint
 """
 
 import copy
+import io
 import json
+import re
 from collections.abc import Callable, Mapping
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Union, cast
@@ -20,6 +22,11 @@ from litellm.anthropic_beta_headers_manager import (
 from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
 from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
+from litellm.llms.base_llm.base_utils import (
+    RESPONSE_FORMAT_ENVELOPE_NAME_KEYS,
+    ResponseFormatSchema,
+    response_format_envelope_contents,
+)
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -57,7 +64,7 @@ from litellm.types.utils import (
 
 from ...base import BaseLLM
 from ..common_utils import AnthropicError, process_anthropic_headers
-from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig
+from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig, JsonModeRequest
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -69,6 +76,67 @@ def _loads_stream_chunk(payload: str) -> dict[str, object]:
     return json.loads(payload)
 
 
+_EnvelopeVerdict = Literal["undecided", "pass_through", "hold"]
+_FIRST_OBJECT_KEY: Final = re.compile(r'^\s*\{\s*"((?:[^"\\]|\\.)*)"\s*:')
+_OBJECT_KEY_PREFIX: Final = re.compile(r'^\s*(?:\{\s*(?:"(?:[^"\\]|\\.)*\\?(?:"\s*)?)?)?$')
+_ENVELOPE_NAME_KEY_PATTERN: Final = "|".join(sorted(RESPONSE_FORMAT_ENVELOPE_NAME_KEYS))
+_FIRST_NAME_VALUE: Final = re.compile(r'^\s*\{\s*"(?:' + _ENVELOPE_NAME_KEY_PATTERN + r')"\s*:\s*"((?:[^"\\]|\\.)*)"')
+_NAME_VALUE_PREFIX: Final = re.compile(
+    r'^\s*\{\s*"(?:' + _ENVELOPE_NAME_KEY_PATTERN + r')"\s*:\s*(?:"(?:[^"\\]|\\.)*\\?)?$'
+)
+_STRING_VALUE_THEN_KEY: Final = re.compile(r'^"(?:[^"\\]|\\.)*"\s*,\s*"((?:[^"\\]|\\.)*)"\s*:')
+_UNDECIDED_PREFIX_LIMIT: Final = 4096
+
+
+def _key_after_a_string_verdict(string_value_start: str, schema: ResponseFormatSchema) -> _EnvelopeVerdict:
+    next_key: Final = _STRING_VALUE_THEN_KEY.match(string_value_start)
+    if next_key is None:
+        return "undecided"
+    return "pass_through" if next_key.group(1) in schema.property_names else "hold"
+
+
+def _first_value_verdict(after_first_key: str, schema: ResponseFormatSchema) -> _EnvelopeVerdict:
+    value_start: Final = after_first_key.lstrip()
+    if not value_start:
+        return "undecided"
+    if value_start.startswith("{"):
+        return "hold"
+    if value_start.startswith('"') and schema.required_names:
+        return _key_after_a_string_verdict(value_start, schema)
+    return "pass_through"
+
+
+def _prefix_verdict(buffered_json: str, schema: ResponseFormatSchema, tool_name: str) -> _EnvelopeVerdict:
+    first_key: Final = _FIRST_OBJECT_KEY.match(buffered_json)
+    if first_key is None:
+        return "undecided" if _OBJECT_KEY_PREFIX.match(buffered_json) else "pass_through"
+    if first_key.group(1) in RESPONSE_FORMAT_ENVELOPE_NAME_KEYS:
+        first_value: Final = _FIRST_NAME_VALUE.match(buffered_json)
+        if first_value is not None and first_value.group(1) == tool_name:
+            return "hold"
+        if first_value is None and _NAME_VALUE_PREFIX.match(buffered_json):
+            return "undecided"
+    if first_key.group(1) in schema.property_names or not schema.property_names or schema.accepts_a_lone_undeclared_key:
+        return "pass_through"
+    return _first_value_verdict(buffered_json[first_key.end() :], schema)
+
+
+def _held_answer(held: str, schema: ResponseFormatSchema) -> str:
+    try:
+        payload: Final[object] = json.loads(held)
+    except (json.JSONDecodeError, RecursionError):
+        return held
+    contents: Final = response_format_envelope_contents(payload, schema, RESPONSE_FORMAT_TOOL_NAME)
+    return held if contents is None else json.dumps(contents)
+
+
+def _envelope_verdict(buffered_json: str, schema: ResponseFormatSchema, tool_name: str) -> _EnvelopeVerdict:
+    verdict: Final = _prefix_verdict(buffered_json, schema, tool_name)
+    if verdict == "undecided" and len(buffered_json) > _UNDECIDED_PREFIX_LIMIT:
+        return "hold"
+    return verdict
+
+
 async def make_call(
     client: AsyncHTTPHandler | None,
     api_base: str,
@@ -81,6 +149,7 @@ async def make_call(
     json_mode: bool,
     speed: str | None = None,
     tool_name_reverse_map: dict[str, str] | None = None,
+    response_format_schema: ResponseFormatSchema | None = None,
 ) -> tuple["ModelResponseIterator", httpx.Headers]:
     if client is None:
         client = litellm.module_level_aclient
@@ -116,6 +185,7 @@ async def make_call(
         json_mode=json_mode,
         speed=speed,
         tool_name_reverse_map=tool_name_reverse_map,
+        response_format_schema=response_format_schema,
     )
 
     # LOGGING
@@ -141,6 +211,7 @@ def make_sync_call(
     json_mode: bool,
     speed: str | None = None,
     tool_name_reverse_map: dict[str, str] | None = None,
+    response_format_schema: ResponseFormatSchema | None = None,
 ) -> tuple["ModelResponseIterator", httpx.Headers]:
     if client is None:
         client = litellm.module_level_client  # re-use a module level client
@@ -184,6 +255,7 @@ def make_sync_call(
         json_mode=json_mode,
         speed=speed,
         tool_name_reverse_map=tool_name_reverse_map,
+        response_format_schema=response_format_schema,
     )
 
     # LOGGING
@@ -241,6 +313,9 @@ class AnthropicChatCompletion(BaseLLM):
             tool_name_reverse_map=(
                 litellm_params.get(ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY) if isinstance(litellm_params, dict) else None
             ),
+            response_format_schema=(
+                JsonModeRequest.model_validate(optional_params).tool_schema() if json_mode and optional_params else None
+            ),
         )
         streamwrapper: Final = CustomStreamWrapper(
             completion_stream=completion_stream,
@@ -485,6 +560,9 @@ class AnthropicChatCompletion(BaseLLM):
                         if isinstance(litellm_params, dict)
                         else None
                     ),
+                    response_format_schema=(
+                        JsonModeRequest.model_validate(optional_params).tool_schema() if json_mode else None
+                    ),
                 )
                 return CustomStreamWrapper(
                     completion_stream=completion_stream,
@@ -550,6 +628,7 @@ class ModelResponseIterator:
         json_mode: bool | None = False,
         speed: str | None = None,
         tool_name_reverse_map: dict[str, str] | None = None,
+        response_format_schema: ResponseFormatSchema | None = None,
     ):
         self.streaming_response = streaming_response
         self.response_iterator = self.streaming_response
@@ -573,6 +652,9 @@ class ModelResponseIterator:
         self.is_response_format_tool: bool = False
         # Track if we've converted any response_format tools (affects finish_reason)
         self.converted_response_format_tool: bool = False
+        self.response_format_schema = response_format_schema
+        self._response_format_buffer: io.StringIO = io.StringIO()
+        self._response_format_verdict: _EnvelopeVerdict = "undecided"
 
         # For handling partial JSON chunks from fragmentation
         # See: https://github.com/BerriAI/litellm/issues/17473
@@ -955,6 +1037,8 @@ class ModelResponseIterator:
                             except (json.JSONDecodeError, TypeError):
                                 pass
                         self._current_server_tool_id = None
+                if self.is_response_format_tool:
+                    text = self._flush_response_format_buffer()
                 # Reset response_format tool tracking when block stops
                 self.is_response_format_tool = False
                 # Reset current content block type
@@ -1072,6 +1156,10 @@ class ModelResponseIterator:
             tool_name: Final = tool_use.get("function", {}).get("name", "")
             self.is_response_format_tool = tool_name == RESPONSE_FORMAT_TOOL_NAME
 
+        if self.is_response_format_tool and self.response_format_schema is not None:
+            self.converted_response_format_tool = True
+            return self._stream_response_format_fragment(tool_use, self.response_format_schema), None
+
         # Convert tool to content if we're tracking a response_format tool
         if self.is_response_format_tool:
             message: Final = AnthropicConfig.convert_tool_response_to_message(tool_calls=[tool_use])
@@ -1083,6 +1171,31 @@ class ModelResponseIterator:
 
         return text, tool_use
 
+    def _stream_response_format_fragment(
+        self, tool_use: ChatCompletionToolCallChunk, schema: ResponseFormatSchema
+    ) -> str:
+        if tool_use.get("id") is not None:
+            self._response_format_buffer = io.StringIO()
+            self._response_format_verdict = "undecided"
+            return ""
+        fragment: Final = tool_use.get("function", {}).get("arguments") or ""
+        if self._response_format_verdict == "pass_through":
+            return fragment
+        self._response_format_buffer.write(fragment)
+        if self._response_format_verdict == "hold":
+            return ""
+        buffered: Final = self._response_format_buffer.getvalue()
+        self._response_format_verdict = _envelope_verdict(buffered, schema, RESPONSE_FORMAT_TOOL_NAME)
+        return buffered if self._response_format_verdict == "pass_through" else ""
+
+    def _flush_response_format_buffer(self) -> str:
+        if self.response_format_schema is None or self._response_format_verdict == "pass_through":
+            return ""
+        held: Final = self._response_format_buffer.getvalue()
+        self._response_format_buffer = io.StringIO()
+        self._response_format_verdict = "undecided"
+        return _held_answer(held, self.response_format_schema) if held else ""
+
     def _handle_message_delta(self, chunk: dict) -> tuple[str, Usage | None, dict[str, Any] | None]:
         """
         Handle message_delta event for finish_reason, usage, and container.

litellm/llms/anthropic/chat/transformation.py

@@ -6,7 +6,7 @@ from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast
 
 import httpx
-from pydantic import BaseModel, ValidationError
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 from typing_extensions import ReadOnly, TypedDict
 
 import litellm
@@ -35,7 +35,13 @@ from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import
     place_mid_conversation_system,
     split_leading_system_run,
 )
-from litellm.llms.base_llm.base_utils import type_to_response_format_param
+from litellm.llms.base_llm.base_utils import (
+    ResponseFormatSchema,
+    as_json_object,
+    response_format_schema,
+    type_to_response_format_param,
+    unwrap_response_format_envelope,
+)
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.types.llms.anthropic import (
     ANTHROPIC_ADVISOR_TOOL_TYPE,
@@ -1746,7 +1752,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         else:
             _input_schema.update(cast(AnthropicInputSchema, json_schema))
 
-        _tool: Final = AnthropicMessagesTool(name=RESPONSE_FORMAT_TOOL_NAME, input_schema=_input_schema)
+        _tool: Final = AnthropicMessagesTool(
+            name=RESPONSE_FORMAT_TOOL_NAME,
+            input_schema=_input_schema,
+        )
         return _tool
 
     def should_strip_billing_metadata(self) -> bool:
@@ -2199,6 +2208,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         self,
         json_mode: bool | None,
         tool_calls: list[ChatCompletionToolCallChunk],
+        response_format_schema: ResponseFormatSchema | None = None,
     ) -> tuple[
         LitellmMessage | None,
         list[ChatCompletionToolCallChunk],
@@ -2218,11 +2228,15 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             json_tool: Final = tool_calls[json_indices[0]]
             if json_tool.get("function", {}).get("arguments") is None:
                 return None, tool_calls, None
-            _message: Final = AnthropicConfig.convert_tool_response_to_message(tool_calls=[json_tool])
+            _message: Final = AnthropicConfig.convert_tool_response_to_message(
+                tool_calls=[json_tool], response_format_schema=response_format_schema
+            )
             return _message, [], None
 
         first_json: Final = tool_calls[json_indices[0]]
-        json_msg: Final = AnthropicConfig.convert_tool_response_to_message([first_json])
+        json_msg: Final = AnthropicConfig.convert_tool_response_to_message(
+            [first_json], response_format_schema=response_format_schema
+        )
         extra_content: Final[str | None] = json_msg.content if json_msg is not None else None
         filtered_tools: Final = [t for i, t in enumerate(tool_calls) if i not in json_indices]
         return None, filtered_tools, extra_content
@@ -2658,6 +2672,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         prefix_prompt: str | None = None,
         speed: str | None = None,
         tool_name_reverse_map: dict[str, str] | None = None,
+        response_format_schema: ResponseFormatSchema | None = None,
     ):
         _hidden_params: Final[dict] = {}
         _hidden_params["additional_headers"] = process_anthropic_headers(dict(raw_response.headers))
@@ -2715,6 +2730,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         json_mode_message, tool_calls_for_message, json_extra_content = self._resolve_json_mode_non_streaming(
             json_mode=json_mode,
             tool_calls=tool_calls,
+            response_format_schema=response_format_schema,
         )
         merged_text = text_content or ""
         if json_extra_content:
@@ -2823,12 +2839,14 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             prefix_prompt=prefix_prompt,
             speed=speed,
             tool_name_reverse_map=tool_name_reverse_map,
+            response_format_schema=JsonModeRequest.model_validate(optional_params).tool_schema() if json_mode else None,
         )
         return model_response
 
     @staticmethod
     def _convert_tool_response_to_message(
         tool_calls: list[ChatCompletionToolCallChunk],
+        response_format_schema: ResponseFormatSchema | None = None,
     ) -> LitellmMessage | None:
         """
         In JSON mode, Anthropic API returns JSON schema as a tool call, we need to convert it to a message to follow the OpenAI format
@@ -2836,28 +2854,32 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         """
         ## HANDLE JSON MODE - anthropic returns single function call
         json_mode_content_str: Final[str | None] = tool_calls[0]["function"].get("arguments")
+        if json_mode_content_str is None:
+            return None
         try:
-            if json_mode_content_str is not None:
-                args: Final = json.loads(json_mode_content_str)
-                if isinstance(args, dict) and (values := args.get("values")) is not None:
-                    _message = litellm.Message(content=json.dumps(values))
-                    return _message
-                else:
-                    # a lot of the times the `values` key is not present in the tool response
-                    # relevant issue: https://github.com/BerriAI/litellm/issues/6741
-                    _message = litellm.Message(content=json.dumps(args))
-                    return _message
-        except json.JSONDecodeError:
+            payload: Final[object] = json.loads(json_mode_content_str)
+        except (json.JSONDecodeError, RecursionError):
             # json decode error does occur, return the original tool response str
             return litellm.Message(content=json_mode_content_str)
-        return None
+        if response_format_schema is not None:
+            return litellm.Message(
+                content=json.dumps(
+                    unwrap_response_format_envelope(payload, response_format_schema, RESPONSE_FORMAT_TOOL_NAME)
+                )
+            )
+        # a lot of the times the `values` key is not present in the tool response
+        # relevant issue: https://github.com/BerriAI/litellm/issues/6741
+        payload_object: Final = as_json_object(payload)
+        values: Final = None if payload_object is None else payload_object.get("values")
+        return litellm.Message(content=json.dumps(payload if values is None else values))
 
     @classmethod
     def convert_tool_response_to_message(
         cls,
         tool_calls: list[ChatCompletionToolCallChunk],  # mutable-ok: mirrors override contract
+        response_format_schema: ResponseFormatSchema | None = None,
     ) -> LitellmMessage | None:
-        return cls._convert_tool_response_to_message(tool_calls)
+        return cls._convert_tool_response_to_message(tool_calls, response_format_schema)
 
     def get_error_class(self, error_message: str, status_code: int, headers: dict | httpx.Headers) -> BaseLLMException:
         return AnthropicError(
@@ -2867,6 +2889,23 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         )
 
 
+_REQUEST_TOOLS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
+
+
+class JsonModeRequest(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    tools: object = None
+
+    def tool_schema(self) -> ResponseFormatSchema | None:
+        try:
+            tools: Final = _REQUEST_TOOLS.validate_python(self.tools)
+        except ValidationError:
+            return None
+        json_tool: Final = next((tool for tool in tools if tool.get("name") == RESPONSE_FORMAT_TOOL_NAME), None)
+        return None if json_tool is None else response_format_schema(json_tool.get("input_schema"))
+
+
 def _valid_user_id(user_id: str) -> bool:
     """
     Validate that user_id is not an email or phone number.

litellm/llms/anthropic/count_tokens/token_counter.py

@@ -4,6 +4,8 @@ Anthropic Token Counter implementation using the CountTokens API.
 
 from typing import Any, Final
 
+from pydantic import JsonValue
+
 from litellm._logging import verbose_logger
 from litellm.exceptions import AuthenticationError
 from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
@@ -26,12 +28,12 @@ class AnthropicTokenCounter(BaseTokenCounter):
     async def count_tokens(
         self,
         model_to_use: str,
-        messages: list[dict[str, Any]] | None,
+        messages: list[dict[str, JsonValue]] | None,
         contents: list[dict[str, object]] | None,
         deployment: dict[str, Any] | None = None,
         request_model: str = "",
-        tools: list[dict[str, Any]] | None = None,
-        system: Any | None = None,
+        tools: list[dict[str, JsonValue]] | None = None,
+        system: JsonValue = None,
     ) -> TokenCountResponse | None:
         """
         Count tokens using Anthropic's CountTokens API.

litellm/llms/anthropic/files/transformation.py

@@ -14,7 +14,7 @@ Anthropic Files API endpoints:
 
 import calendar
 import time
-from collections.abc import Mapping
+from collections.abc import Iterable, Mapping
 from typing import Final, cast
 
 import httpx
@@ -49,6 +49,7 @@ ANTHROPIC_FILES_BETA_HEADER: Final = "files-api-2025-04-14"
 ANTHROPIC_MESSAGE_BATCH_ID_PREFIX: Final = "msgbatch_"
 
 _JSON_OBJECT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_ITEMS: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True))
 
 
 class AnthropicFilesConfig(BaseFilesConfig):
@@ -284,9 +285,9 @@ class AnthropicFilesConfig(BaseFilesConfig):
             "last_id": "..."
         }
         """
-        response_json: Final = raw_response.json()
-        files_data: Final = response_json.get("data", [])
-        return [self._parse_anthropic_file(f) for f in files_data]
+        response_json: Final = _JSON_OBJECT.validate_python(raw_response.json())
+        files_data: Final = _JSON_ITEMS.validate_python(response_json.get("data", []))
+        return [self._parse_anthropic_file(_JSON_OBJECT.validate_python(f)) for f in files_data]
 
     def transform_file_content_request(
         self,

litellm/llms/anthropic/pass_through/adapters/transformation.py

@@ -1347,7 +1347,7 @@ class LiteLLMAnthropicMessagesAdapter:
         self,
         choices: list[Choices],
         tool_name_mapping: dict[str, str] | None = None,
-    ) -> list[dict[str, Any]]:
+    ) -> list[dict[str, object]]:
         new_content: Final[list[dict[str, Any]]] = []
         for choice, compaction_blocks in (
             (choice, _compaction_blocks(_optional_attr(choice.message, "provider_specific_fields")))

litellm/llms/azure/azure.py

@@ -13,6 +13,7 @@ from openai import (
     BadRequestError,
     OpenAI,
 )
+from pydantic import BaseModel
 
 import litellm
 from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
@@ -41,6 +42,7 @@ from ..base import BaseLLM
 from ..openai.common_utils import (
     build_output_token_limit_response,
     is_output_token_limit_error,
+    model_dump_provider_response,
 )
 from .common_utils import (
     AzureOpenAIError,
@@ -352,12 +354,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 headers, response = self.make_sync_azure_openai_chat_completion_request(
                     azure_client=azure_client, data=data, timeout=timeout
                 )
-                if isinstance(response, str):
+                if not isinstance(response, BaseModel):
                     raise AzureOpenAIError(
                         status_code=500,
-                        message=f"Unexpected string response from Azure: {response[:500]}",
+                        message=f"Unexpected response from Azure: {response!r:.500}",
                     )
-                stringified_response: Final = response.model_dump()
+                stringified_response: Final = model_dump_provider_response(response)
                 ## LOGGING
                 logging_obj.post_call(
                     input=messages,
@@ -447,12 +449,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             )
             logging_obj.model_call_details["response_headers"] = headers
 
-            if isinstance(response, str):
+            if not isinstance(response, BaseModel):
                 raise AzureOpenAIError(
                     status_code=500,
-                    message=f"Unexpected string response from Azure: {response[:500]}",
+                    message=f"Unexpected response from Azure: {response!r:.500}",
                 )
-            stringified_response: Final = response.model_dump()
+            stringified_response: Final = model_dump_provider_response(response)
             logging_obj.post_call(
                 input=data["messages"],
                 api_key=api_key,

litellm/llms/azure_ai/anthropic/count_tokens/handler.py

@@ -4,9 +4,10 @@ Azure AI Anthropic CountTokens API handler.
 Uses httpx for HTTP requests with Azure authentication.
 """
 
-from typing import Any, Final
+from typing import Final
 
 import httpx
+from pydantic import JsonValue
 
 import litellm
 from litellm._logging import verbose_logger
@@ -28,14 +29,14 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
     async def handle_count_tokens_request(
         self,
         model: str,
-        messages: list[dict[str, Any]],
+        messages: list[dict[str, JsonValue]],
         api_key: str,
         api_base: str,
         litellm_params: dict[str, object] | None = None,
         timeout: float | httpx.Timeout | None = None,
-        tools: list[dict[str, Any]] | None = None,
+        tools: list[dict[str, JsonValue]] | None = None,
         system: object = None,
-    ) -> dict[str, Any]:
+    ) -> dict[str, object]:
         """
         Handle a CountTokens request using httpx with Azure authentication.
 

litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py

@@ -5,6 +5,8 @@ Azure AI Anthropic Token Counter implementation using the CountTokens API.
 import os
 from typing import Any, Final
 
+from pydantic import JsonValue
+
 from litellm._logging import verbose_logger
 from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
     AzureAIAnthropicCountTokensHandler,
@@ -28,11 +30,11 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
     async def count_tokens(
         self,
         model_to_use: str,
-        messages: list[dict[str, object]] | None,
+        messages: list[dict[str, JsonValue]] | None,
         contents: list[dict[str, object]] | None,
         deployment: dict[str, Any] | None = None,
         request_model: str = "",
-        tools: list[dict[str, object]] | None = None,
+        tools: list[dict[str, JsonValue]] | None = None,
         system: object | None = None,
     ) -> TokenCountResponse | None:
         """
@@ -86,12 +88,14 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
             )
 
             if result is not None:
-                return TokenCountResponse(
-                    total_tokens=result.get("input_tokens", 0),
-                    request_model=request_model,
-                    model_used=model_to_use,
-                    tokenizer_type="azure_ai_anthropic_api",
-                    original_response=result,
+                return TokenCountResponse.model_validate(
+                    {
+                        "total_tokens": result.get("input_tokens", 0),
+                        "request_model": request_model,
+                        "model_used": model_to_use,
+                        "tokenizer_type": "azure_ai_anthropic_api",
+                        "original_response": result,
+                    }
                 )
         except AnthropicError as e:
             verbose_logger.warning(

litellm/llms/azure_ai/anthropic/handler.py

@@ -11,6 +11,7 @@ import httpx
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.anthropic.chat.handler import AnthropicChatCompletion
+from litellm.llms.anthropic.chat.transformation import JsonModeRequest
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -162,6 +163,9 @@ class AzureAnthropicChatCompletion(AnthropicChatCompletion):
                     logging_obj=logging_obj,
                     timeout=timeout,
                     json_mode=json_mode,
+                    response_format_schema=(
+                        JsonModeRequest.model_validate(optional_params).tool_schema() if json_mode else None
+                    ),
                 )
                 from litellm.llms.anthropic.common_utils import (
                     process_anthropic_headers,

litellm/llms/azure_ai/image_edit/flux2_transformation.py

@@ -113,7 +113,7 @@ class AzureFoundryFlux2ImageEditConfig(OpenAIImageEditConfig):
                 for index, reference_image in enumerate(images, start=1)
             }
         )
-        request_body: Final[dict[str, Any]] = {
+        request_body: Final[dict[str, object]] = {
             "prompt": prompt,
             "model": model,
             **reference_images,

litellm/llms/base_llm/base_utils.py

@@ -3,13 +3,15 @@ Utility functions for base LLM classes.
 """
 
 import copy
+import itertools
 import json
 from abc import ABC, abstractmethod
-from collections.abc import Mapping
+from collections.abc import Iterator, Mapping
+from dataclasses import dataclass
 from typing import Any, Final
 
 from openai.lib import _parsing, _pydantic  # pyright: ignore[reportPrivateUsage]  # SDK parser internals
-from pydantic import BaseModel
+from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
@@ -157,6 +159,136 @@ def convert_tool_response_to_message(
 _convert_tool_response_to_message = convert_tool_response_to_message
 
 
+@dataclass(frozen=True, slots=True)
+class ResponseFormatSchema:
+    property_names: frozenset[str]
+    required_names: frozenset[str] = frozenset()
+    allows_undeclared_keys: bool = False
+
+    @property
+    def accepts_a_lone_undeclared_key(self) -> bool:
+        return self.allows_undeclared_keys and not self.required_names
+
+
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, JsonValue])
+_NAMES: Final = TypeAdapter(frozenset[str])
+
+
+def as_json_object(value: object) -> Mapping[str, JsonValue] | None:
+    try:
+        return _JSON_OBJECT.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def _names(value: object) -> frozenset[str]:
+    try:
+        return _NAMES.validate_python(value)
+    except ValidationError:
+        return frozenset()
+
+
+def _allows_undeclared_keys(document: Mapping[str, JsonValue]) -> bool:
+    additional_properties: Final = document.get("additionalProperties")
+    return (
+        additional_properties is True
+        or as_json_object(additional_properties) is not None
+        or bool(as_json_object(document.get("patternProperties")))
+    )
+
+
+def response_format_schema(value: object) -> ResponseFormatSchema | None:
+    document: Final = as_json_object(value)
+    if document is None:
+        return None
+    properties: Final = as_json_object(document.get("properties", {}))
+    if properties is None:
+        return None
+    return ResponseFormatSchema(
+        property_names=frozenset(properties),
+        required_names=_names(document.get("required", ())),
+        allows_undeclared_keys=_allows_undeclared_keys(document),
+    )
+
+
+RESPONSE_FORMAT_ENVELOPE_NAME_KEYS: Final = frozenset({"name", "tool_name"})
+
+
+@dataclass(frozen=True, slots=True)
+class _EnvelopeLayer:
+    inner: Mapping[str, JsonValue]
+    names_the_tool: bool
+
+
+def _json_object_or_encoded_object(value: JsonValue) -> Mapping[str, JsonValue] | None:
+    if not isinstance(value, str):
+        return as_json_object(value)
+    try:
+        return as_json_object(json.loads(value))
+    except (json.JSONDecodeError, RecursionError):
+        return None
+
+
+def _label_keys(payload_object: Mapping[str, JsonValue], schema: ResponseFormatSchema) -> frozenset[str]:
+    if not schema.required_names:
+        return frozenset()
+    declared: Final = schema.property_names | schema.required_names
+    return frozenset(key for key, value in payload_object.items() if isinstance(value, str) and key not in declared)
+
+
+def _envelope_layer(
+    payload_object: Mapping[str, JsonValue], schema: ResponseFormatSchema, tool_name: str
+) -> _EnvelopeLayer | None:
+    name_keys: Final = frozenset(
+        key for key in RESPONSE_FORMAT_ENVELOPE_NAME_KEYS if payload_object.get(key) == tool_name
+    )
+    label_keys: Final = name_keys or _label_keys(payload_object, schema)
+    wrapper_keys: Final = tuple(key for key in payload_object if key not in label_keys)
+    if len(wrapper_keys) != 1 or wrapper_keys[0] in schema.property_names:
+        return None
+    wrapped: Final = payload_object[wrapper_keys[0]]
+    inner: Final = _json_object_or_encoded_object(wrapped) if name_keys else as_json_object(wrapped)
+    return None if inner is None else _EnvelopeLayer(inner=inner, names_the_tool=bool(name_keys))
+
+
+def _envelope_layers(
+    payload_object: Mapping[str, JsonValue], schema: ResponseFormatSchema, tool_name: str
+) -> Iterator[_EnvelopeLayer]:
+    layer = _envelope_layer(payload_object, schema, tool_name)  # rebind-ok: the recursion gate needs a loop
+    while layer is not None:
+        yield layer
+        layer = _envelope_layer(layer.inner, schema, tool_name)
+
+
+def _schema_object_inside_envelope(
+    payload_object: Mapping[str, JsonValue], schema: ResponseFormatSchema, tool_name: str
+) -> Mapping[str, JsonValue] | None:
+    layers: Final = _envelope_layers(payload_object, schema, tool_name)
+    candidates: Final = layers if schema.property_names else itertools.islice(layers, 1)
+    return next((layer.inner for layer in candidates if _holds_the_schema_object(layer, schema)), None)
+
+
+def _holds_the_schema_object(layer: _EnvelopeLayer, schema: ResponseFormatSchema) -> bool:
+    if layer.names_the_tool:
+        return True
+    return not schema.accepts_a_lone_undeclared_key and not schema.property_names.isdisjoint(layer.inner)
+
+
+def response_format_envelope_contents(payload: object, schema: ResponseFormatSchema, tool_name: str) -> object | None:
+    payload_object: Final = as_json_object(payload)
+    if payload_object is None:
+        return None
+    schema_object: Final = _schema_object_inside_envelope(payload_object, schema, tool_name)
+    if schema_object is not None:
+        return schema_object
+    return None if "values" in schema.property_names else payload_object.get("values")
+
+
+def unwrap_response_format_envelope(payload: object, schema: ResponseFormatSchema, tool_name: str) -> object:
+    contents: Final = response_format_envelope_contents(payload, schema, tool_name)
+    return payload if contents is None else contents
+
+
 def _dict_to_response_format_helper(response_format: dict, ref_template: str | None = None) -> dict:
     if ref_template is not None and response_format.get("type") == "json_schema":
         # Deep copy to avoid modifying original

litellm/llms/base_llm/managed_resources/isolation.py

@@ -10,6 +10,7 @@ they created. Callers with no admin role and no identifying ids at all
 are denied so an empty user_id can never select an unscoped query.
 """
 
+from collections.abc import Mapping, Sequence
 from typing import Any, Final
 
 from litellm.types.proxy.auth.user_api_key_auth import (
@@ -46,7 +47,7 @@ def resolve_resource_owner_id(
     return None
 
 
-def build_list_page(items: list[Any], has_more: bool = False) -> dict[str, Any]:
+def build_list_page(items: list[Any], has_more: bool = False) -> dict[str, object]:
     """Build the OpenAI-style paginated list response shape used by managed
     file/batch/vector-store listings. ``first_id`` and ``last_id`` are
     sourced from each item's ``.id`` attribute."""
@@ -61,7 +62,7 @@ def build_list_page(items: list[Any], has_more: bool = False) -> dict[str, Any]:
 
 def build_owner_filter(
     user_api_key_dict: UserAPIKeyAuth,
-) -> dict[str, Any] | None:
+) -> dict[str, str | Sequence[Mapping[str, str]]] | None:
     """Return a Prisma `where` fragment that scopes a managed-resource listing
     to records the caller is allowed to see.
 

litellm/llms/bedrock/chat/invoke_handler.py

@@ -1,9 +1,10 @@
+import json
 import types
 from collections.abc import AsyncIterator, Iterator, Mapping
 from typing import TYPE_CHECKING, Final, cast
 
 import httpx
-from pydantic import TypeAdapter
+from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm import verbose_logger
@@ -16,6 +17,8 @@ from litellm.litellm_core_utils.optional_imports import ensure_optional_import
 from litellm.llms.anthropic.chat.handler import (
     ModelResponseIterator as AnthropicModelResponseIterator,
 )
+from litellm.llms.anthropic.chat.transformation import JsonModeRequest
+from litellm.llms.base_llm.base_utils import ResponseFormatSchema
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -174,15 +177,34 @@ class AmazonCohereChatConfig:
         return optional_params
 
 
+_REQUEST_BODY: Final = TypeAdapter(Mapping[str, object])
+
+
+def response_format_schema_from_request(data: str, json_mode: bool | None) -> ResponseFormatSchema | None:
+    if not json_mode:
+        return None
+    try:
+        body: Final = _REQUEST_BODY.validate_python(json.loads(data))
+    except (json.JSONDecodeError, ValidationError):
+        return None
+    return JsonModeRequest.model_validate(body).tool_schema()
+
+
 def _stream_decoder(
     bedrock_invoke_provider: litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL | None,
     *,
     model: str,
     json_mode: bool | None,
     sync_stream: bool,
+    response_format_schema: ResponseFormatSchema | None = None,
 ) -> "AWSEventStreamDecoder":
     if bedrock_invoke_provider == "anthropic":
-        return AmazonAnthropicClaudeStreamDecoder(model=model, sync_stream=sync_stream, json_mode=json_mode)
+        return AmazonAnthropicClaudeStreamDecoder(
+            model=model,
+            sync_stream=sync_stream,
+            json_mode=json_mode,
+            response_format_schema=response_format_schema,
+        )
     if bedrock_invoke_provider == "deepseek_r1":
         return AmazonDeepSeekR1StreamDecoder(model=model, sync_stream=sync_stream)
     if bedrock_invoke_provider == "moonshot":
@@ -250,7 +272,11 @@ async def make_call(
             )
         else:
             decoder: Final = _stream_decoder(
-                bedrock_invoke_provider, model=model, json_mode=json_mode, sync_stream=False
+                bedrock_invoke_provider,
+                model=model,
+                json_mode=json_mode,
+                sync_stream=False,
+                response_format_schema=response_format_schema_from_request(data, json_mode),
             )
             completion_stream = decoder.aiter_bytes(
                 response.aiter_bytes(chunk_size=stream_chunk_size), response_headers=response.headers
@@ -341,7 +367,11 @@ def make_sync_call(
             )
         else:
             decoder: Final = _stream_decoder(
-                bedrock_invoke_provider, model=model, json_mode=json_mode, sync_stream=True
+                bedrock_invoke_provider,
+                model=model,
+                json_mode=json_mode,
+                sync_stream=True,
+                response_format_schema=response_format_schema_from_request(data, json_mode),
             )
             completion_stream = decoder.iter_bytes(
                 response.iter_bytes(chunk_size=stream_chunk_size), response_headers=response.headers
@@ -878,6 +908,7 @@ class AmazonAnthropicClaudeStreamDecoder(AWSEventStreamDecoder):
         model: str,
         sync_stream: bool,
         json_mode: bool | None = None,
+        response_format_schema: ResponseFormatSchema | None = None,
     ) -> None:
         """
         Child class of AWSEventStreamDecoder that handles the streaming response from the Anthropic family of models
@@ -889,6 +920,7 @@ class AmazonAnthropicClaudeStreamDecoder(AWSEventStreamDecoder):
             streaming_response=None,
             sync_stream=sync_stream,
             json_mode=json_mode,
+            response_format_schema=response_format_schema,
         )
 
     def _chunk_parser(self, chunk_data: dict) -> ModelResponseStream:

litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen2_transformation.py

@@ -7,9 +7,11 @@ The main difference is in the response format: Qwen2 uses "text" field while Qwe
 Qwen2 + Invoke API Tutorial: https://docs.aws.amazon.com/bedrock/latest/userguide/invoke-imported-model.html
 """
 
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.llms.bedrock.chat.invoke_transformations.amazon_qwen3_transformation import (
     AmazonQwen3Config,
@@ -23,6 +25,9 @@ from litellm.types.utils import ModelResponse, Usage
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_TEXT: Final = TypeAdapter(str, config=ConfigDict(strict=True, hide_input_in_errors=True))
+
 
 class AmazonQwen2Config(AmazonQwen3Config):
     """
@@ -54,10 +59,10 @@ class AmazonQwen2Config(AmazonQwen3Config):
         Qwen2 uses "text" field, but we also support "generation" field for compatibility.
         """
         try:
-            response_data: Final = raw_response.json()
+            response_data: Final = _JSON_OBJECT.validate_python(raw_response.json())
 
             # Extract the generated text - Qwen2 uses "text" field, but also support "generation" for compatibility
-            generated_text = response_data.get("generation", "") or response_data.get("text", "")
+            generated_text = _TEXT.validate_python(response_data.get("generation", "") or response_data.get("text", ""))
 
             # Clean up the response (remove assistant start token if present)
             generated_text = generated_text.removeprefix("<|im_start|>assistant\n")
@@ -71,14 +76,16 @@ class AmazonQwen2Config(AmazonQwen3Config):
 
             # Set usage information if available in response
             if "usage" in response_data:
-                usage_data: Final = response_data["usage"]
+                usage_data: Final = _JSON_OBJECT.validate_python(response_data["usage"])
                 setattr(
                     model_response,
                     "usage",
-                    Usage(
-                        prompt_tokens=usage_data.get("prompt_tokens", 0),
-                        completion_tokens=usage_data.get("completion_tokens", 0),
-                        total_tokens=usage_data.get("total_tokens", 0),
+                    Usage.model_validate(
+                        {
+                            "prompt_tokens": usage_data.get("prompt_tokens", 0),
+                            "completion_tokens": usage_data.get("completion_tokens", 0),
+                            "total_tokens": usage_data.get("total_tokens", 0),
+                        }
                     ),
                 )
 

litellm/llms/bedrock/chat/invoke_transformations/amazon_qwen3_transformation.py

@@ -10,6 +10,7 @@ from collections.abc import Mapping
 from typing import TYPE_CHECKING, Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.llms.base_llm.chat.transformation import BaseConfig
 from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation import (
@@ -22,6 +23,9 @@ from litellm.types.utils import ModelResponse, Usage
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_TEXT: Final = TypeAdapter(str, config=ConfigDict(strict=True, hide_input_in_errors=True))
+
 
 class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
     """
@@ -179,10 +183,10 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
         Transform Qwen3 Bedrock response to OpenAI format
         """
         try:
-            response_data: Final = raw_response.json()
+            response_data: Final = _JSON_OBJECT.validate_python(raw_response.json())
 
             # Extract the generated text - Qwen3 uses "generation" field
-            generated_text = response_data.get("generation", "")
+            generated_text = _TEXT.validate_python(response_data.get("generation", ""))
 
             # Clean up the response (remove assistant start token if present)
             generated_text = generated_text.removeprefix("<|im_start|>assistant\n")
@@ -196,14 +200,16 @@ class AmazonQwen3Config(AmazonInvokeConfig, BaseConfig):
 
             # Set usage information if available in response
             if "usage" in response_data:
-                usage_data: Final = response_data["usage"]
+                usage_data: Final = _JSON_OBJECT.validate_python(response_data["usage"])
                 setattr(
                     model_response,
                     "usage",
-                    Usage(
-                        prompt_tokens=usage_data.get("prompt_tokens", 0),
-                        completion_tokens=usage_data.get("completion_tokens", 0),
-                        total_tokens=usage_data.get("total_tokens", 0),
+                    Usage.model_validate(
+                        {
+                            "prompt_tokens": usage_data.get("prompt_tokens", 0),
+                            "completion_tokens": usage_data.get("completion_tokens", 0),
+                            "total_tokens": usage_data.get("total_tokens", 0),
+                        }
                     ),
                 )
 

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -51,6 +51,10 @@ _GUARDRAIL_CONFIG_EXPECTED_FORMAT: Final = (
     "{'guardrailIdentifier': str, 'guardrailVersion': str, 'trace': 'enabled'|'disabled'|'enabled_full'}"
 )
 
+_INVOKE_PROVIDERS: Final = TypeAdapter(tuple[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, ...]).validate_python(
+    get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL)
+)
+
 
 def _bedrock_invoke_guardrail_headers(raw_guardrail_config: object) -> "dict[str, str]":
     try:
@@ -568,7 +572,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         if provider is not None:
             return provider
 
-        for provider in get_args(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL):
+        for provider in _INVOKE_PROVIDERS:
             if provider in model:
                 return provider
         return None

litellm/llms/brave/search/transformation.py

@@ -6,15 +6,20 @@ Documentation: https://api-dashboard.search.brave.com/app/documentation/web-sear
 from __future__ import annotations
 
 import re
+from collections.abc import Iterable, Mapping
 from datetime import datetime, timezone
 from typing import Final, Literal, TypedDict
 
 import httpx
 from dateutil import parser
+from pydantic import ConfigDict, TypeAdapter
 
 _ISO_YMD: Final = re.compile(r"^\s*\d{4}[-/]\d{1,2}[-/]\d{1,2}\s*$")
 _UNIX_TIMESTAMP: Final = re.compile(r"^\s*-?\d+(\.\d+)?\s*$")
 BRAVE_SECTIONS: Final = ["web", "discussions", "faqs", "faq", "news", "videos"]
+_JSON_VALUE: Final = TypeAdapter(object)
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True))
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.search.transformation import (
@@ -26,7 +31,7 @@ from litellm.secret_managers.main import get_secret_str
 
 
 def to_yyyy_mm_dd(
-    s: str | float | None,
+    s: object,
     *,
     dayfirst: bool = False,
     yearfirst: bool = False,
@@ -246,7 +251,7 @@ class BraveSearchConfig(BaseSearchConfig):
         """
         Transform Brave Search API response to LiteLLM unified SearchResponse format.
         """
-        response_json: Final = raw_response.json()
+        response_body: Final = _JSON_VALUE.validate_python(raw_response.json())
 
         # Transform results to SearchResult objects
         results: Final[list[SearchResult]] = []
@@ -254,9 +259,11 @@ class BraveSearchConfig(BaseSearchConfig):
         query_params: Final = raw_response.request.url.params if raw_response.request else {}
         sections_to_process: Final = self._sections_from_params(dict(query_params))
         max_results: Final = max(1, min(int(query_params.get("count", 20)), 20))
+        response_json: Final = _JSON_OBJECT.validate_python(response_body)
 
         for section in sections_to_process:
-            for result in response_json.get(section, {}).get("results", []):
+            section_json = _JSON_OBJECT.validate_python(response_json.get(section, {}))
+            for raw_result in _JSON_VALUES.validate_python(section_json.get("results", [])):
                 # Because the `max_results`/`count` parameters do not affect
                 # the number of "discussion", "faq", "news", or "videos"
                 # results, we need to manually limit the number of results
@@ -264,18 +271,21 @@ class BraveSearchConfig(BaseSearchConfig):
                 if len(results) >= max_results:
                     break
 
+                result = _JSON_OBJECT.validate_python(raw_result)
                 title = result.get("title", "")
                 url = result.get("url", "")
                 snippet = result.get("description", "")
                 date = to_yyyy_mm_dd(result.get("page_age") or result.get("age"))
                 last_updated = to_yyyy_mm_dd(result.get("fetched_content_timestamp", ""))
 
-                search_result = SearchResult(
-                    title=title,
-                    url=url,
-                    snippet=snippet,
-                    date=date,
-                    last_updated=last_updated,
+                search_result = SearchResult.model_validate(
+                    {
+                        "title": title,
+                        "url": url,
+                        "snippet": snippet,
+                        "date": date,
+                        "last_updated": last_updated,
+                    }
                 )
 
                 results.append(search_result)

litellm/llms/cohere/embed/transformation.py

@@ -10,10 +10,11 @@ Convers
 Docs - https://docs.cohere.com/v2/reference/embed
 """
 
-from collections.abc import Sized
+from collections.abc import Iterable, Mapping, Sized
 from typing import Final, Protocol, cast
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 import litellm
 from litellm import COHERE_DEFAULT_EMBEDDING_INPUT_TYPE
@@ -30,6 +31,9 @@ from litellm.utils import is_base64_encoded
 
 from ..common_utils import CohereError
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_EMBEDDINGS_BY_TYPE: Final = TypeAdapter(Mapping[str, Iterable[object]], config=ConfigDict(hide_input_in_errors=True))
+
 
 class _SupportsEncode(Protocol):
     """Tokenizer handle: the embedding usage path only encodes text to measure its token length."""
@@ -198,7 +202,8 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig):
                 'usage'
             }
         """
-        embeddings: Final = response_json["embeddings"]
+        payload: Final = _JSON_OBJECT.validate_python(response_json)
+        embeddings: Final = _EMBEDDINGS_BY_TYPE.validate_python(payload["embeddings"])
         output_data: Final = []
         for k, embedding_list in embeddings.items():
             for idx, embedding in enumerate(embedding_list):
@@ -213,7 +218,7 @@ class CohereEmbeddingConfig(BaseEmbeddingConfig):
         setattr(
             model_response,
             "usage",
-            self._calculate_usage(input, encoding, response_json.get("meta", {})),
+            self._calculate_usage(input, encoding, dict(_JSON_OBJECT.validate_python(payload.get("meta", {})))),
         )
 
         return model_response

litellm/llms/cometapi/embed/transformation.py

@@ -2,9 +2,11 @@
 CometAPI Embedding API support - OpenAI compatible
 """
 
+from collections.abc import Mapping
 from typing import Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
@@ -15,6 +17,9 @@ from litellm.types.utils import EmbeddingResponse, Usage
 
 from ..common_utils import CometAPIException
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_INT: Final = TypeAdapter(int)
+
 
 class CometAPIEmbeddingConfig(BaseEmbeddingConfig):
     """
@@ -136,9 +141,10 @@ class CometAPIEmbeddingConfig(BaseEmbeddingConfig):
         model_response.data = raw_response_json.get("data")
         model_response.object = raw_response_json.get("object")
 
+        usage_payload: Final = _JSON_OBJECT.validate_python(raw_response_json.get("usage", {}))
         usage: Final = Usage(
-            prompt_tokens=raw_response_json.get("usage", {}).get("prompt_tokens", 0),
-            total_tokens=raw_response_json.get("usage", {}).get("total_tokens", 0),
+            prompt_tokens=_INT.validate_python(usage_payload.get("prompt_tokens", 0) or 0),
+            total_tokens=_INT.validate_python(usage_payload.get("total_tokens", 0) or 0),
         )
 
         model_response.usage = usage

litellm/llms/dashscope/embed/transformation.py

@@ -147,7 +147,7 @@ class DashScopeEmbeddingConfig(BaseEmbeddingConfig):
         )
 
         if "error" in response_json:
-            error: Final = response_json["error"]
+            error: Final[object] = response_json["error"]
             message: Final = error.get("message", str(error)) if isinstance(error, dict) else str(error)
             raise DashScopeError(
                 status_code=raw_response.status_code,

litellm/llms/databricks/chat/transformation.py

@@ -475,7 +475,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
             # Move message-level cache_control into a content block when content is a string.
             if "cache_control" in _message and isinstance(_message.get("content"), str):
                 _message = self._move_cache_control_into_string_content_block(_message)
-            _sanitize_empty_content(cast(dict[str, Any], _message))
+            _sanitize_empty_content(cast(dict[str, object], _message))
             if _is_bare_assistant_message(_message):
                 continue
             new_messages.append(_message)
@@ -504,7 +504,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
         """
         content: Final = message.get("content")
         # Create new message with cache_control moved into content block
-        transformed_message: Final = cast(dict[str, Any], message.copy())
+        transformed_message: Final = cast(dict[str, object], message.copy())
         cache_control: Final = transformed_message.pop("cache_control")
         transformed_message["content"] = [
             {
@@ -697,6 +697,10 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
         model_response.model = f"{_custom_llm_provider}/{_response_model}"
         model_response.id = completion_response["id"]
         model_response.created = completion_response["created"]
+        if "system_fingerprint" in completion_response:
+            model_response.system_fingerprint = (  # rebind-ok: fills the caller's model_response like id above
+                completion_response["system_fingerprint"]
+            )
         setattr(model_response, "usage", Usage(**completion_response["usage"]))
         served_tier: Final = completion_response.get("service_tier")
         if isinstance(served_tier, str) and served_tier:

litellm/llms/dataforseo/search/transformation.py

@@ -4,9 +4,11 @@ Calls DataForSEO SERP API to search the web.
 DataForSEO API Reference: https://docs.dataforseo.com/v3/serp/google/organic/live/advanced/?bash
 """
 
+from collections.abc import Iterable, Mapping
 from typing import Final, Literal
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.constants import DEFAULT_DATAFORSEO_LOCATION_CODE
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -17,6 +19,9 @@ from litellm.llms.base_llm.search.transformation import (
 )
 from litellm.secret_managers.main import get_secret_str
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True))
+
 
 class DataForSEOSearchConfig(BaseSearchConfig):
     """
@@ -201,18 +206,21 @@ class DataForSEOSearchConfig(BaseSearchConfig):
             if task.get("status_code") == 20000 and "result" in task:
                 # Result is an array, take first element
                 if len(task["result"]) > 0:
-                    result: Final = task["result"][0]
+                    result: Final = _JSON_OBJECT.validate_python(task["result"][0])
 
                     # Items contain the actual search results
-                    for item in result.get("items", []):
+                    for raw_item in _JSON_VALUES.validate_python(result.get("items", [])):
+                        item = _JSON_OBJECT.validate_python(raw_item)
                         # Only process organic search results
                         if item.get("type") == "organic":
-                            search_result = SearchResult(
-                                title=item.get("title", ""),
-                                url=item.get("url", ""),
-                                snippet=item.get("description", ""),
-                                date=None,  # DataForSEO doesn't provide date in standard response
-                                last_updated=None,
+                            search_result = SearchResult.model_validate(
+                                {
+                                    "title": item.get("title", ""),
+                                    "url": item.get("url", ""),
+                                    "snippet": item.get("description", ""),
+                                    "date": None,  # DataForSEO doesn't provide date in standard response
+                                    "last_updated": None,
+                                }
                             )
                             results.append(search_result)
 

litellm/llms/deepagents/harness/transformation.py

@@ -14,7 +14,7 @@ import json
 from collections.abc import Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any, Final, Protocol
 
 from litellm.harness.options import DeepAgentsOptions
 from litellm.harness.types import (
@@ -73,7 +73,7 @@ _APPROVAL_DECISIONS: Final = ("approve", "reject")
 
 def chat_model_kwargs(
     ctx: SessionContext,
-) -> dict[str, Any]:  # mutable-ok: ChatLiteLLM constructor kwargs, splatted as **kwargs
+) -> dict[str, object]:  # mutable-ok: ChatLiteLLM constructor kwargs, splatted as **kwargs
     """ChatLiteLLM constructor kwargs for gateway or SDK mode."""
     if not ctx.model:
         raise ValueError("Harness.DEEPAGENTS needs model=")
@@ -107,7 +107,7 @@ def blocked_tools(permissions: str, disable_tools: Sequence[str]) -> frozenset[s
 
 def interrupt_config(
     permissions: str, blocked: frozenset[str]
-) -> dict[str, Any] | None:  # mutable-ok: deepagents create_deep_agent(interrupt_on=) takes a dict
+) -> dict[str, dict[str, list[str]]] | None:  # mutable-ok: deepagents create_deep_agent(interrupt_on=) takes a dict
     """interrupt_on for permissions='ask': approve/reject every mutating built-in."""
     if permissions != "ask":
         return None
@@ -221,9 +221,14 @@ def _message_events(message: object, skip_tools: frozenset[str]) -> tuple[Event,
     return ()
 
 
+class InterruptLike(Protocol):
+    @property
+    def id(self) -> str: ...
+
+
 def interrupts_in(
     update: object,
-) -> list[Any]:  # mutable-ok: returns a list; existing callers/tests compare it to list literals
+) -> list[InterruptLike]:  # mutable-ok: returns a list; existing callers/tests compare it to list literals
     if not isinstance(update, Mapping):
         return []  # mutable-ok: returns a list; existing callers/tests compare it to list literals
     found = update.get("__interrupt__")
@@ -260,7 +265,7 @@ def approval_requests(
     return list(kept)  # mutable-ok: list return; callers/tests compare to lists
 
 
-def decision(allowed: bool, reason: str) -> dict[str, Any]:  # mutable-ok: LangGraph resume payload (HITL decision dict)
+def decision(allowed: bool, reason: str) -> dict[str, str]:  # mutable-ok: LangGraph resume payload (HITL decision dict)
     if allowed:
         return {"type": "approve"}  # mutable-ok: LangGraph resume payload (HITL decision dict)
     return {  # mutable-ok: LangGraph HITL decision
@@ -273,7 +278,7 @@ def decision(allowed: bool, reason: str) -> dict[str, Any]:  # mutable-ok: LangG
 class TurnState:
     """Mutable state across the stream passes of one turn."""
 
-    interrupts: tuple[Any, ...] = ()
+    interrupts: tuple[InterruptLike, ...] = ()
 
 
 class DeepAgentsHarnessConfig(BaseHarnessConfig):

litellm/llms/duckduckgo/search/transformation.py

@@ -4,10 +4,12 @@ Calls DuckDuckGo's Instant Answer API to search the web.
 DuckDuckGo API Reference: https://duckduckgo.com/api
 """
 
+from collections.abc import Iterable, Mapping
 from typing import Final, Literal, TypedDict
 from urllib.parse import urlencode
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.search.transformation import (
@@ -17,6 +19,9 @@ from litellm.llms.base_llm.search.transformation import (
 )
 from litellm.secret_managers.main import get_secret_str
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True))
+
 
 class _DuckDuckGoSearchRequestRequired(TypedDict):
     """Required fields for DuckDuckGo Search API request."""
@@ -159,7 +164,7 @@ class DuckDuckGoSearchConfig(BaseSearchConfig):
         Returns:
             SearchResponse with standardized format
         """
-        response_json: Final = raw_response.json()
+        response_json: Final = _JSON_OBJECT.validate_python(raw_response.json())
 
         # Extract max_results from the request URL params
         query_params: Final = raw_response.request.url.params if raw_response.request else {}
@@ -178,17 +183,19 @@ class DuckDuckGoSearchConfig(BaseSearchConfig):
 
         # Check if there's an Abstract with URL
         if response_json.get("AbstractURL") and response_json.get("AbstractText"):
-            abstract_result: Final = SearchResult(
-                title=response_json.get("Heading", ""),
-                url=response_json.get("AbstractURL", ""),
-                snippet=response_json.get("AbstractText", ""),
-                date=None,
-                last_updated=None,
+            abstract_result: Final = SearchResult.model_validate(
+                {
+                    "title": response_json.get("Heading", ""),
+                    "url": response_json.get("AbstractURL", ""),
+                    "snippet": response_json.get("AbstractText", ""),
+                    "date": None,
+                    "last_updated": None,
+                }
             )
             results.append(abstract_result)
 
         # Process RelatedTopics
-        related_topics: Final = response_json.get("RelatedTopics", [])
+        related_topics: Final = _JSON_VALUES.validate_python(response_json.get("RelatedTopics", []))
         for topic in related_topics:
             # Stop if we've reached max_results
             if max_results is not None and len(results) >= max_results:

litellm/llms/exa_ai/search/transformation.py

@@ -4,9 +4,11 @@ Calls Exa AI's /search endpoint to search the web.
 Exa AI API Reference: https://docs.exa.ai/reference/search
 """
 
+from collections.abc import Iterable, Mapping
 from typing import Final, TypedDict
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.search.transformation import (
@@ -16,6 +18,10 @@ from litellm.llms.base_llm.search.transformation import (
 )
 from litellm.secret_managers.main import get_secret_str
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True))
+_STRS: Final = TypeAdapter(Iterable[str], config=ConfigDict(hide_input_in_errors=True))
+
 
 class _ExaAISearchRequestRequired(TypedDict):
     """Required fields for Exa AI Search API request."""
@@ -177,20 +183,23 @@ class ExaAISearchConfig(BaseSearchConfig):
         Returns:
             SearchResponse with standardized format
         """
-        response_json: Final = raw_response.json()
+        response_json: Final = _JSON_OBJECT.validate_python(raw_response.json())
 
         # Transform results to SearchResult objects
         results: Final = []
-        for result in response_json.get("results", []):
-            search_result = SearchResult(
-                title=result.get("title", ""),
-                url=result.get("url", ""),
-                snippet=result.get("text")
-                or "\n\n".join(result.get("highlights") or [])
-                or result.get("summary")
-                or "",
-                date=result.get("publishedDate"),  # ISO 8601 datetime string
-                last_updated=None,  # Exa AI doesn't provide last_updated in response
+        for raw_result in _JSON_VALUES.validate_python(response_json.get("results", [])):
+            result = _JSON_OBJECT.validate_python(raw_result)
+            search_result = SearchResult.model_validate(
+                {
+                    "title": result.get("title", ""),
+                    "url": result.get("url", ""),
+                    "snippet": result.get("text")
+                    or "\n\n".join(_STRS.validate_python(result.get("highlights") or []))
+                    or result.get("summary")
+                    or "",
+                    "date": result.get("publishedDate"),  # ISO 8601 datetime string
+                    "last_updated": None,  # Exa AI doesn't provide last_updated in response
+                }
             )
             results.append(search_result)
 

litellm/llms/fal_ai/image_edit/transformation.py

@@ -7,6 +7,7 @@ from typing import TYPE_CHECKING, Final, Protocol, runtime_checkable
 
 import httpx
 from httpx._types import RequestFiles
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.images.utils import ImageEditRequestUtils
 from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig
@@ -23,6 +24,8 @@ from litellm.types.utils import FileTypes, ImageResponse
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 
+_JSON_VALUE: Final = TypeAdapter(object)
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
 DEFAULT_BASE_URL: Final[str] = "https://fal.run"
 EDIT_SUFFIX: Final[str] = "/edit"
 SUPPORTED_OPENAI_PARAMS: Final[tuple[str, ...]] = ("background", "mask", "n", "quality", "size")
@@ -163,7 +166,7 @@ class FalAIImageEditConfig(BaseImageEditConfig):
         logging_obj: "LiteLLMLoggingObj",
     ) -> ImageResponse:
         try:
-            response_json: Final = raw_response.json()
+            response_json: Final = _JSON_VALUE.validate_python(raw_response.json())
         except Exception as e:
             raise self.get_error_class(
                 error_message=f"Error parsing Fal AI image edit response: {e}",
@@ -171,5 +174,7 @@ class FalAIImageEditConfig(BaseImageEditConfig):
                 headers=raw_response.headers,
             )
         model_response: Final = ImageResponse()
-        model_response.data = list(fal_images_to_image_objects(response_json.get("images", ())))
+        model_response.data = list(
+            fal_images_to_image_objects(_JSON_OBJECT.validate_python(response_json).get("images", ()))
+        )
         return model_response

litellm/llms/fal_ai/image_generation/bria_transformation.py

@@ -1,6 +1,8 @@
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
 from litellm.types.utils import ImageObject, ImageResponse
@@ -15,6 +17,9 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObj = Any
 
+_JSON_VALUE: Final = TypeAdapter(object)
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+
 
 class FalAIBriaConfig(FalAIBaseConfig):
     """
@@ -206,7 +211,7 @@ class FalAIBriaConfig(FalAIBaseConfig):
         }
         """
         try:
-            response_data: Final = raw_response.json()
+            response_data: Final = _JSON_VALUE.validate_python(raw_response.json())
         except Exception as e:
             raise self.get_error_class(
                 error_message=f"Error transforming image generation response: {e}",
@@ -218,7 +223,7 @@ class FalAIBriaConfig(FalAIBaseConfig):
             model_response.data = []
 
         # Handle Bria response format - uses "image" (singular) not "images"
-        image_data: Final = response_data.get("image")
+        image_data: Final = _JSON_OBJECT.validate_python(response_data).get("image")
         if image_data and isinstance(image_data, dict):
             model_response.data.append(
                 ImageObject(

litellm/llms/fal_ai/image_generation/imagen4_transformation.py

@@ -1,6 +1,8 @@
+from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final, cast
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
 from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams
@@ -16,6 +18,9 @@ if TYPE_CHECKING:
 else:
     LiteLLMLoggingObj = Any
 
+_JSON_VALUE: Final = TypeAdapter(object)
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+
 
 class FalAIImagen4Config(FalAIBaseConfig):
     """
@@ -203,7 +208,7 @@ class FalAIImagen4Config(FalAIBaseConfig):
         }
         """
         try:
-            response_data: Final = raw_response.json()
+            response_data: Final = _JSON_VALUE.validate_python(raw_response.json())
         except Exception as e:
             raise self.get_error_class(
                 error_message=f"Error transforming image generation response: {e}",
@@ -215,7 +220,8 @@ class FalAIImagen4Config(FalAIBaseConfig):
             model_response.data = []
 
         # Handle Imagen4 response format
-        images: Final = response_data.get("images", [])
+        response_object: Final = _JSON_OBJECT.validate_python(response_data)
+        images: Final = response_object.get("images", [])
         if isinstance(images, list):
             for image_data in images:
                 if isinstance(image_data, dict):
@@ -239,7 +245,7 @@ class FalAIImagen4Config(FalAIBaseConfig):
             hidden_params: Final = cast(  # cast-ok: preserve mapping operations on dynamic response metadata
                 dict[str, object], getattr(model_response, HIDDEN_PARAMS_ATTR)
             )
-            if "seed" in response_data:
-                hidden_params["seed"] = response_data["seed"]
+            if "seed" in response_object:
+                hidden_params["seed"] = response_object["seed"]
 
         return model_response

litellm/llms/fal_ai/image_generation/transformation.py

@@ -2,7 +2,7 @@ from collections.abc import Mapping
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
-from pydantic import TypeAdapter
+from pydantic import ConfigDict, TypeAdapter
 from typing_extensions import ReadOnly, TypedDict
 
 from litellm.llms.base_llm.image_generation.transformation import (
@@ -31,6 +31,8 @@ class FalImageProviderSpecificFields(TypedDict, total=False):
 
 
 _FAL_IMAGE_DATA: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+_JSON_VALUE: Final = TypeAdapter(object)
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
 
 
 def fal_images_to_image_objects(images: object) -> tuple[ImageObject, ...]:
@@ -124,7 +126,7 @@ class FalAIBaseConfig(BaseImageGenerationConfig):
         Transform the image generation response to the litellm image response
         """
         try:
-            response_data: Final = raw_response.json()
+            response_data: Final = _JSON_VALUE.validate_python(raw_response.json())
         except Exception as e:
             raise self.get_error_class(
                 error_message=f"Error transforming image generation response: {e}",
@@ -134,7 +136,9 @@ class FalAIBaseConfig(BaseImageGenerationConfig):
         if not model_response.data:
             model_response.data = []
 
-        model_response.data.extend(fal_images_to_image_objects(response_data.get("images", ())))
+        model_response.data.extend(
+            fal_images_to_image_objects(_JSON_OBJECT.validate_python(response_data).get("images", ()))
+        )
         return model_response
 
 

litellm/llms/fastcrw/search/transformation.py

@@ -8,9 +8,11 @@ or cloud). The search response uses the Firecrawl-compatible envelope
 fastCRW API Reference: https://fastcrw.com/docs/rest-api
 """
 
+from collections.abc import Iterable, Mapping
 from typing import Final, TypedDict
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.search.transformation import (
@@ -20,6 +22,9 @@ from litellm.llms.base_llm.search.transformation import (
 )
 from litellm.secret_managers.main import get_secret_str
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True))
+
 
 class _FastCRWSearchRequestRequired(TypedDict):
     """Required fields for fastCRW Search API request."""
@@ -157,21 +162,24 @@ class FastCRWSearchConfig(BaseSearchConfig):
         Returns:
             SearchResponse with standardized format
         """
-        response_json: Final = raw_response.json()
+        response_json: Final = _JSON_OBJECT.validate_python(raw_response.json())
 
         results: Final = []
 
         data: Final = response_json.get("data", [])
 
         if isinstance(data, list):
-            for result in data:
+            for raw_result in _JSON_VALUES.validate_python(data):
+                result = _JSON_OBJECT.validate_python(raw_result)
                 snippet = result.get("markdown") or result.get("description", "")
-                search_result = SearchResult(
-                    title=result.get("title", ""),
-                    url=result.get("url", ""),
-                    snippet=snippet,
-                    date=None,
-                    last_updated=None,
+                search_result = SearchResult.model_validate(
+                    {
+                        "title": result.get("title", ""),
+                        "url": result.get("url", ""),
+                        "snippet": snippet,
+                        "date": None,
+                        "last_updated": None,
+                    }
                 )
                 results.append(search_result)
 

litellm/llms/firecrawl/search/transformation.py

@@ -4,9 +4,11 @@ Calls Firecrawl's /search endpoint to search the web.
 Firecrawl API Reference: https://docs.firecrawl.dev/api-reference/endpoint/search
 """
 
+from collections.abc import Iterable, Mapping
 from typing import Final, TypedDict
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.search.transformation import (
@@ -16,6 +18,9 @@ from litellm.llms.base_llm.search.transformation import (
 )
 from litellm.secret_managers.main import get_secret_str
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True))
+
 
 class _FirecrawlSearchRequestRequired(TypedDict):
     """Required fields for Firecrawl Search API request."""
@@ -170,7 +175,7 @@ class FirecrawlSearchConfig(BaseSearchConfig):
         Returns:
             SearchResponse with standardized format
         """
-        response_json: Final = raw_response.json()
+        response_json: Final = _JSON_OBJECT.validate_python(raw_response.json())
 
         # Transform results to SearchResult objects
         results: Final = []
@@ -179,44 +184,54 @@ class FirecrawlSearchConfig(BaseSearchConfig):
 
         if isinstance(data, list):
             # Self-hosted Firecrawl (v1) format: data is a flat list of results
-            for result in data:
+            for raw_result in _JSON_VALUES.validate_python(data):
+                result = _JSON_OBJECT.validate_python(raw_result)
                 snippet = result.get("markdown") or result.get("description", "")
-                search_result = SearchResult(
-                    title=result.get("title", ""),
-                    url=result.get("url", ""),
-                    snippet=snippet,
-                    date=None,
-                    last_updated=None,
+                search_result = SearchResult.model_validate(
+                    {
+                        "title": result.get("title", ""),
+                        "url": result.get("url", ""),
+                        "snippet": snippet,
+                        "date": None,
+                        "last_updated": None,
+                    }
                 )
                 results.append(search_result)
         elif isinstance(data, dict):
             # Firecrawl Cloud (v2) format: data is a dict with web/news keys
-            web_results: Final = data.get("web", [])
+            sections: Final = _JSON_OBJECT.validate_python(data)
+            web_results: Final = _JSON_VALUES.validate_python(sections.get("web", []))
 
-            for result in web_results:
+            for raw_result in web_results:
+                result = _JSON_OBJECT.validate_python(raw_result)
                 # Use markdown if available, otherwise fall back to description
                 snippet = result.get("markdown") or result.get("description", "")
 
-                search_result = SearchResult(
-                    title=result.get("title", ""),
-                    url=result.get("url", ""),
-                    snippet=snippet,
-                    date=None,
-                    last_updated=None,
+                search_result = SearchResult.model_validate(
+                    {
+                        "title": result.get("title", ""),
+                        "url": result.get("url", ""),
+                        "snippet": snippet,
+                        "date": None,
+                        "last_updated": None,
+                    }
                 )
                 results.append(search_result)
 
             # Process news results if available (they have date field)
-            news_results: Final = data.get("news", [])
-            for result in news_results:
+            news_results: Final = _JSON_VALUES.validate_python(sections.get("news", []))
+            for raw_result in news_results:
+                result = _JSON_OBJECT.validate_python(raw_result)
                 snippet = result.get("markdown") or result.get("snippet", "")
 
-                search_result = SearchResult(
-                    title=result.get("title", ""),
-                    url=result.get("url", ""),
-                    snippet=snippet,
-                    date=result.get("date"),  # News results include date
-                    last_updated=None,
+                search_result = SearchResult.model_validate(
+                    {
+                        "title": result.get("title", ""),
+                        "url": result.get("url", ""),
+                        "snippet": snippet,
+                        "date": result.get("date"),  # News results include date
+                        "last_updated": None,
+                    }
                 )
                 results.append(search_result)
 

litellm/llms/fireworks_ai/chat/transformation.py

@@ -1,8 +1,9 @@
 import json
-from collections.abc import AsyncIterator, Iterator, Mapping
+from collections.abc import AsyncIterator, Iterable, Iterator, Mapping
 from typing import TYPE_CHECKING, Final, Literal, cast
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
@@ -51,6 +52,10 @@ from ..common_utils import (
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
 
+_JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True))
+_STR: Final = TypeAdapter(str)
+
 
 def _map_reasoning_effort(value: object) -> object:
     effort: Final[object] = cast(Mapping[str, object], value).get("effort") if isinstance(value, Mapping) else value
@@ -757,7 +762,8 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
 
         additional_headers: Final = get_response_headers(raw_response_headers)
 
-        response: Final = ModelResponse(**completion_response)
+        completion_fields: Final = _JSON_DICT.validate_python(completion_response)
+        response: Final = ModelResponse(**completion_fields)
 
         if response.model is not None:
             response.model = "fireworks_ai/" + response.model
@@ -771,7 +777,7 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
 
         response.hidden_params = {
             "additional_headers": additional_headers,
-            **_extract_fireworks_hidden_params(completion_response),
+            **_extract_fireworks_hidden_params(completion_fields),
         }
 
         return response
@@ -832,9 +838,9 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig):
                 f"Failed to fetch models from Fireworks AI. Status code: {response.status_code}, Response: {response.json()}"
             )
 
-        models: Final = response.json()["models"]
+        models: Final = _JSON_OBJECTS.validate_python(_JSON_DICT.validate_python(response.json())["models"])
 
-        return ["fireworks_ai/" + model["name"] for model in models]
+        return ["fireworks_ai/" + _STR.validate_python(model["name"]) for model in models]
 
     @staticmethod
     def get_api_key(api_key: str | None = None) -> str | None:

litellm/llms/gemini/image_generation/transformation.py

@@ -1,4 +1,4 @@
-from collections.abc import Iterable, Mapping
+from collections.abc import Iterable, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
@@ -35,6 +35,10 @@ else:
 
 _JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True))
 _JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_CONTAINER: Final[TypeAdapter[Mapping[str, object] | str | Sequence[object]]] = TypeAdapter(
+    Mapping[str, object] | str | Sequence[object], config=ConfigDict(hide_input_in_errors=True)
+)
 
 
 class GoogleImageGenConfig(BaseImageGenerationConfig):
@@ -199,19 +203,22 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
         # Handle different response formats based on model
         if is_gemini_image_model(model):
             # Gemini Flash Image Preview models return in candidates format
-            candidates: Final = response_data.get("candidates", [])
+            response_object: Final = _JSON_OBJECT.validate_python(response_data)
+            candidates: Final = _JSON_CONTAINER.validate_python(response_object.get("candidates", []))
             for candidate in candidates:
-                content = candidate.get("content", {})
-                parts = content.get("parts", [])
-                for part in parts:
+                content = _JSON_OBJECT.validate_python(_JSON_OBJECT.validate_python(candidate).get("content", {}))
+                parts = _JSON_CONTAINER.validate_python(content.get("parts", []))
+                for raw_part in parts:
+                    part = raw_part if isinstance(raw_part, str) else _JSON_CONTAINER.validate_python(raw_part)
                     # Look for inlineData with image
                     if "inlineData" in part:
-                        inline_data = part["inlineData"]
+                        part_object = _JSON_OBJECT.validate_python(part)
+                        inline_data = _JSON_CONTAINER.validate_python(part_object["inlineData"])
                         if "data" in inline_data:
-                            thought_sig = part.get("thoughtSignature")
+                            thought_sig = part_object.get("thoughtSignature")
                             model_response.data.append(
                                 ImageObject(
-                                    b64_json=inline_data["data"],
+                                    b64_json=_JSON_OBJECT.validate_python(inline_data)["data"],
                                     url=None,
                                     provider_specific_fields=(
                                         {"thought_signature": thought_sig} if thought_sig else None
@@ -224,12 +231,14 @@ class GoogleImageGenConfig(BaseImageGenerationConfig):
                 model_response.usage = transform_gemini_image_usage(
                     _JSON_DICT.validate_python(response_data["usageMetadata"])
                 )
-            web_search_requests: Final = get_gemini_image_web_search_requests(response_data)
+            web_search_requests: Final = get_gemini_image_web_search_requests(response_object)
             if web_search_requests and model_response.usage is not None:
                 setattr(model_response.usage, "web_search_requests", web_search_requests)
         else:
             # Original Imagen format - predictions with generated images
-            predictions: Final = _JSON_OBJECTS.validate_python(response_data.get("predictions", []))
+            predictions: Final = _JSON_OBJECTS.validate_python(
+                _JSON_OBJECT.validate_python(response_data).get("predictions", [])
+            )
             for prediction in predictions:
                 # Google AI returns base64 encoded images in the prediction
                 model_response.data.append(

litellm/llms/google_pse/search/transformation.py

@@ -4,9 +4,11 @@ Calls Google Programmable Search Engine (PSE) API to search the web.
 Google PSE API Reference: https://developers.google.com/custom-search/v1/reference/rest/v1/cse/list
 """
 
+from collections.abc import Iterable, Mapping
 from typing import Final, Literal, TypedDict
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.search.transformation import (
@@ -16,6 +18,9 @@ from litellm.llms.base_llm.search.transformation import (
 )
 from litellm.secret_managers.main import get_secret_str
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True))
+
 
 class _GooglePSESearchRequestRequired(TypedDict):
     """Required fields for Google PSE Search API request."""
@@ -239,17 +244,20 @@ class GooglePSESearchConfig(BaseSearchConfig):
         Returns:
             SearchResponse with standardized format
         """
-        response_json: Final = raw_response.json()
+        response_json: Final = _JSON_OBJECT.validate_python(raw_response.json())
 
         # Transform results to SearchResult objects
         results: Final = []
-        for item in response_json.get("items", []):
-            search_result = SearchResult(
-                title=item.get("title", ""),
-                url=item.get("link", ""),
-                snippet=item.get("snippet", ""),
-                date=None,  # Google PSE doesn't provide date in standard response
-                last_updated=None,  # Google PSE doesn't provide last_updated in response
+        for raw_item in _JSON_VALUES.validate_python(response_json.get("items", [])):
+            item = _JSON_OBJECT.validate_python(raw_item)
+            search_result = SearchResult.model_validate(
+                {
+                    "title": item.get("title", ""),
+                    "url": item.get("link", ""),
+                    "snippet": item.get("snippet", ""),
+                    "date": None,  # Google PSE doesn't provide date in standard response
+                    "last_updated": None,  # Google PSE doesn't provide last_updated in response
+                }
             )
             results.append(search_result)
 

litellm/llms/infinity/embedding/transformation.py

@@ -1,6 +1,8 @@
+from collections.abc import Mapping
 from typing import Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
@@ -11,6 +13,9 @@ from litellm.types.utils import EmbeddingResponse, Usage
 
 from ..common_utils import InfinityError
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_INT: Final = TypeAdapter(int)
+
 
 class InfinityEmbeddingConfig(BaseEmbeddingConfig):
     """
@@ -124,9 +129,10 @@ class InfinityEmbeddingConfig(BaseEmbeddingConfig):
         model_response.data = raw_response_json.get("data")
         model_response.object = raw_response_json.get("object")
 
+        usage_payload: Final = _JSON_OBJECT.validate_python(raw_response_json.get("usage", {}))
         usage: Final = Usage(
-            prompt_tokens=raw_response_json.get("usage", {}).get("prompt_tokens", 0),
-            total_tokens=raw_response_json.get("usage", {}).get("total_tokens", 0),
+            prompt_tokens=_INT.validate_python(usage_payload.get("prompt_tokens", 0) or 0),
+            total_tokens=_INT.validate_python(usage_payload.get("total_tokens", 0) or 0),
         )
         model_response.usage = usage
         return model_response

litellm/llms/langgraph/chat/transformation.py

@@ -10,7 +10,7 @@ Non-streaming endpoint: POST /runs/wait
 
 import json
 from collections.abc import Mapping
-from typing import TYPE_CHECKING, Any, Final, Optional, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Optional, Union
 
 import httpx
 
@@ -369,7 +369,7 @@ class LangGraphConfig(BaseConfig):
         from litellm.utils import CustomStreamWrapper
 
         if client is None or not isinstance(client, AsyncHTTPHandler):
-            client = get_async_httpx_client(llm_provider=cast(Any, "langgraph"), params={})
+            client = get_async_httpx_client(llm_provider="langgraph", params={})
 
         verbose_logger.debug("Making async streaming request to: %s", api_base)
 

litellm/llms/lemonade/chat/transformation.py

@@ -2,10 +2,12 @@
 Translate from OpenAI's `/v1/chat/completions` to Lemonade's `/v1/chat/completions`
 """
 
+from collections.abc import Iterable, Mapping
 from typing import TYPE_CHECKING, Any, Final
 from urllib.parse import quote
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
@@ -21,6 +23,10 @@ from ...openai_like.chat.transformation import OpenAILikeChatConfig
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer
 
+_JSON_DICT: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_OBJECTS: Final = TypeAdapter(Iterable[Mapping[str, object]], config=ConfigDict(hide_input_in_errors=True))
+_STR: Final = TypeAdapter(str)
+
 
 class LemonadeChatConfig(OpenAILikeChatConfig):
     _DEFAULT_API_KEY = "lemonade"
@@ -55,7 +61,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig):
         response_format: dict | None = None,
         tools: list | None = None,
     ) -> None:
-        locals_: Final = locals().copy()
+        locals_: Final[Mapping[str, object]] = dict(locals())
         for key, value in locals_.items():
             if key != "self" and value is not None:
                 setattr(self.__class__, key, value)
@@ -104,8 +110,8 @@ class LemonadeChatConfig(OpenAILikeChatConfig):
                 f"Failed to fetch models from Lemonade. Status code: {response.status_code}, Response: {response.text}"
             )
 
-        model_list: Final = response.json().get("data", [])
-        return ["lemonade/" + model["id"] for model in model_list]
+        model_list: Final = _JSON_OBJECTS.validate_python(_JSON_DICT.validate_python(response.json()).get("data", []))
+        return ["lemonade/" + _STR.validate_python(model["id"]) for model in model_list]
 
     @staticmethod
     def _get_positive_int(value: Any) -> int | None:
@@ -188,10 +194,11 @@ class LemonadeChatConfig(OpenAILikeChatConfig):
             verbose_logger.debug("LemonadeError: Could not get model info.")
             return self._get_default_model_info(model)
 
-        max_input_tokens: Final = self._get_context_window(model_info)
-        max_output_tokens: Final = self._get_positive_int(model_info.get("max_output_tokens"))
-        max_tokens: Final = self._get_positive_int(model_info.get("max_tokens"))
-        provider_specific_entry: Final = self._get_provider_specific_entry(model_info)
+        model_info_fields: Final = _JSON_DICT.validate_python(model_info)
+        max_input_tokens: Final = self._get_context_window(model_info_fields)
+        max_output_tokens: Final = self._get_positive_int(model_info_fields.get("max_output_tokens"))
+        max_tokens: Final = self._get_positive_int(model_info_fields.get("max_tokens"))
+        provider_specific_entry: Final = self._get_provider_specific_entry(model_info_fields)
 
         model_info_response: Final = self._get_default_model_info(model)
         model_info_response.update(

litellm/llms/linkup/search/transformation.py

@@ -4,9 +4,11 @@ Calls Linkup's /search endpoint to search the web.
 Linkup API Reference: https://docs.linkup.so/pages/documentation/api-reference/endpoint/post-search
 """
 
+from collections.abc import Iterable, Mapping
 from typing import Final, Literal, TypedDict
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.search.transformation import (
@@ -16,6 +18,9 @@ from litellm.llms.base_llm.search.transformation import (
 )
 from litellm.secret_managers.main import get_secret_str
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+_JSON_VALUES: Final = TypeAdapter(Iterable[object], config=ConfigDict(hide_input_in_errors=True))
+
 
 class _LinkupSearchRequestRequired(TypedDict):
     """Required fields for Linkup Search API request."""
@@ -165,35 +170,40 @@ class LinkupSearchConfig(BaseSearchConfig):
         Returns:
             SearchResponse with standardized format
         """
-        response_json: Final = raw_response.json()
+        response_json: Final = _JSON_OBJECT.validate_python(raw_response.json())
 
         # Transform results to SearchResult objects
         results: Final = []
 
         # Process results array
-        raw_results: Final = response_json.get("results", [])
+        raw_results: Final = _JSON_VALUES.validate_python(response_json.get("results", []))
 
-        for result in raw_results:
+        for raw_result in raw_results:
+            result = _JSON_OBJECT.validate_python(raw_result)
             # Handle both text and image result types
             result_type = result.get("type", "text")
 
             if result_type == "text":
-                search_result = SearchResult(
-                    title=result.get("name", ""),
-                    url=result.get("url", ""),
-                    snippet=result.get("content", ""),
-                    date=None,
-                    last_updated=None,
+                search_result = SearchResult.model_validate(
+                    {
+                        "title": result.get("name", ""),
+                        "url": result.get("url", ""),
+                        "snippet": result.get("content", ""),
+                        "date": None,
+                        "last_updated": None,
+                    }
                 )
                 results.append(search_result)
             elif result_type == "image":
                 # For image results, use the URL as both title and snippet if name not provided
-                search_result = SearchResult(
-                    title=result.get("name", result.get("url", "")),
-                    url=result.get("url", ""),
-                    snippet=result.get("content", ""),
-                    date=None,
-                    last_updated=None,
+                search_result = SearchResult.model_validate(
+                    {
+                        "title": result.get("name", result.get("url", "")),
+                        "url": result.get("url", ""),
+                        "snippet": result.get("content", ""),
+                        "date": None,
+                        "last_updated": None,
+                    }
                 )
                 results.append(search_result)
 

litellm/llms/mistral/audio_speech/transformation.py

@@ -11,6 +11,7 @@ from types import MappingProxyType
 from typing import TYPE_CHECKING, Final
 
 import httpx
+from pydantic import ConfigDict, TypeAdapter, ValidationError
 
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.text_to_speech.transformation import (
@@ -23,6 +24,8 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
     from litellm.types.llms.openai import HttpxBinaryResponseContent
 
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
+
 
 class MistralTextToSpeechException(BaseLLMException):
     pass
@@ -143,7 +146,7 @@ class MistralTextToSpeechConfig(BaseTextToSpeechConfig):
         return request_data
 
     def _requested_content_type(self, request: httpx.Request) -> str:
-        request_body: Final = json.loads(request.content or b"{}")
+        request_body: Final = _JSON_OBJECT.validate_python(json.loads(request.content or b"{}"))
         requested_format: Final = request_body.get("response_format")
         if not isinstance(requested_format, str):
             return "audio/mpeg"
@@ -158,7 +161,9 @@ class MistralTextToSpeechConfig(BaseTextToSpeechConfig):
         from litellm.types.llms.openai import HttpxBinaryResponseContent
 
         try:
-            response_json: Final = raw_response.json()
+            response_json: Final = _JSON_OBJECT.validate_python(raw_response.json())
+        except ValidationError:
+            raise
         except (json.JSONDecodeError, ValueError):
             raise MistralTextToSpeechException(
                 status_code=raw_response.status_code,