Merge remote-tracking branch 'origin/main' into litellm_passthrough_auth_guardrails

# Conflicts:
#	litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
#	tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py

.circleci/AGENTS.md

@@ -0,0 +1,7 @@
+# .circleci
+
+The `integration` workflow on `main` runs from a scheduled pipeline in the CircleCI project settings, cron `9 0-8,14-23 * * *` (UTC) on `main` with no pipeline parameters. It has no `triggers:` block in this file because a workflow with one runs only on its schedule, so the API pipelines that set `run_migration_tests` or `routing_parity_base` would stop running it. If main stops getting integration runs, check that the schedule still exists and recreate it with these values
+
+Jobs clone with the `checkout_over_https` command, never the built-in `checkout` step, because CircleCI falls back to the project's SSH checkout key without its GitHub App access, GitHub rejects that key, and the public repo needs no key. It fetches the job's commit with full history and no file contents (`--filter=blob:none`), like the built-in step did, so `scripts/path_filter.sh` can find a merge base
+
+A step that checks out files from another commit fetches that commit with `--no-filter`, as `routing_parity` does, because the clone has no file contents and git would otherwise download each missing file in its own request, which took 210 to 896 seconds for the routing parity base

.circleci/config.yml

@@ -21,19 +21,30 @@ orbs:
   win: circleci/windows@5.0 # Add Windows orb
 
 commands:
-  checkout_migration_source:
+  checkout_over_https:
+    parameters:
+      revision:
+        type: string
+        default: ""
+      shell:
+        type: string
+        default: /bin/bash -eo pipefail
     steps:
       - run:
-          name: Select the requested migration test revision
+          name: Check out the public repository over HTTPS
+          shell: << parameters.shell >>
           environment:
-            MIGRATION_SOURCE_SHA: << pipeline.parameters.migration_source_sha >>
+            REQUESTED_REVISION: << parameters.revision >>
           command: |
-            revision="${MIGRATION_SOURCE_SHA:-$CIRCLE_SHA1}"
+            revision="${REQUESTED_REVISION:-$CIRCLE_SHA1}"
             [[ "$revision" =~ ^[0-9a-f]{40}$ ]] || exit 1
             git init
             git remote add origin https://github.com/BerriAI/litellm.git
-            git fetch --depth 1 origin "$revision"
-            git checkout --detach FETCH_HEAD
+            for attempt in 1 2 3; do
+              git fetch --filter=blob:none origin "$revision" && git checkout --detach FETCH_HEAD && break
+              [[ $attempt -lt 3 ]] || exit 1
+              sleep 5
+            done
   skip_if_unrelated_changes:
     parameters:
       category:
@@ -347,7 +358,7 @@ commands:
       - when:
           condition: << parameters.checkout >>
           steps:
-            - checkout
+            - checkout_over_https
       - setup_google_dns
       - install_uv
       - install_rust
@@ -377,7 +388,8 @@ jobs:
       CARGO_HTTP_MULTIPLEXING: "false"
       CARGO_NET_RETRY: "5"
     steps:
-      - checkout
+      - checkout_over_https:
+          shell: bash.exe
       - run:
           name: Skip job when no windows-release-relevant files changed
           shell: bash.exe
@@ -451,7 +463,7 @@ jobs:
           password: ${DOCKERHUB_PASSWORD}
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - setup_google_dns
       - install_uv
       - install_rust
@@ -482,7 +494,7 @@ jobs:
     working_directory: ~/project
 
     steps:
-      - checkout
+      - checkout_over_https
       - skip_if_unrelated_changes
       - attach_workspace:
           at: ~/project
@@ -559,7 +571,7 @@ jobs:
     resource_class: medium
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -659,7 +671,7 @@ jobs:
     resource_class: large
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -740,7 +752,7 @@ jobs:
     resource_class: large
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - skip_if_unrelated_changes:
           category: client
       - setup_google_dns
@@ -794,7 +806,7 @@ jobs:
       # the rest of the suite: nothing else performs a logout.
       PROXY_LOGOUT_URL: "https://www.example.com"
     steps:
-      - checkout
+      - checkout_over_https
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -940,7 +952,7 @@ jobs:
       # and by migration.serverRootPath.config.ts, which refuses to run without it.
       SERVER_ROOT_PATH: "/litellm"
     steps:
-      - checkout
+      - checkout_over_https
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -1066,11 +1078,12 @@ jobs:
       - when:
           condition: << parameters.migration_qualification >>
           steps:
-            - checkout_migration_source
+            - checkout_over_https:
+                revision: << pipeline.parameters.migration_source_sha >>
       - unless:
           condition: << parameters.migration_qualification >>
           steps:
-            - checkout
+            - checkout_over_https
             - skip_if_unrelated_changes
 
       - run:
@@ -1122,7 +1135,8 @@ jobs:
       MIGRATION_TEST_OUTPUT: /tmp/migration-results
       PYTHONPATH: tests/e2e
     steps:
-      - checkout_migration_source
+      - checkout_over_https:
+          revision: << pipeline.parameters.migration_source_sha >>
       - run:
           name: Generate LiteLLM master key
           command: |
@@ -1197,7 +1211,7 @@ jobs:
     resource_class: medium
     working_directory: ~/project
     steps:
-      - checkout
+      - checkout_over_https
       - skip_if_unrelated_changes
       - attach_workspace:
           at: ~/project
@@ -1371,7 +1385,7 @@ jobs:
     environment:
       UV_PYTHON_DOWNLOADS: never
     steps:
-      - checkout
+      - checkout_over_https
       - install_uv
       - run:
           name: Install the locked shard verifier and coverage dependencies
@@ -1419,7 +1433,7 @@ jobs:
             ROUTING_PARITY_BASE: << pipeline.parameters.routing_parity_base >>
           command: |
             [[ "$ROUTING_PARITY_BASE" =~ ^[0-9a-f]{40}$ ]] || exit 1
-            git fetch --depth 1 origin "$ROUTING_PARITY_BASE"
+            git fetch --depth 1 --no-filter origin "$ROUTING_PARITY_BASE"
             git rm -r -f --quiet litellm enterprise litellm-proxy-extras
             git checkout "$ROUTING_PARITY_BASE" -- litellm enterprise litellm-proxy-extras
             git reset --quiet
@@ -1494,7 +1508,7 @@ jobs:
     environment:
       DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm_test
     steps:
-      - checkout
+      - checkout_over_https
       - skip_if_unrelated_changes
       - setup_litellm_test_deps:
           checkout: false

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -4,9 +4,20 @@
      Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
      "## Affected release" with nothing under it must not appear in the final description -->
 
-<!-- Two or three plain sentences a teammate can read in 10 seconds, written the way you'd explain it in Slack
+## Handwritten 1-3 sentence description
+
+<!-- The PR author rewrites this in their own words before asking for review. Being able to sum up the PR in a few
+     plain sentences proves the author understands what it does, and human-written technical English is far easier
+     for a reviewer to parse than AI-written English
+     An AI agent may draft a starting point here, but always wraps its draft in the make-me-human-readable tags below,
+     backticks included so the tags stay visible on GitHub, so nobody mistakes it for the author's words. The author rewrites it in their own words and deletes the tags.
+     Tags still present at review time mean the summary is AI-written and has not been rewritten yet
      Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
-     No headings, bullets, tables, or code names here. Everything else goes inside the Details block below -->
+     No bullets, tables, or code names here. Everything else goes inside the Details block below -->
+
+`<make-me-human-readable>`
+<blah>
+`</make-me-human-readable>`
 
 <details>
 <summary>Details</summary>

.github/workflows/create-rc-branch.yml

@@ -2,7 +2,7 @@ name: Create RC Branch
 
 on:
   schedule:
-    - cron: "0 3 * * 5"
+    - cron: "0 3 * * 1"
       timezone: "America/Los_Angeles"
   workflow_dispatch:
 

AGENTS.md

@@ -41,6 +41,8 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
 
 If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just drop the section
 
+Run QA proxies with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --num_workers 2 --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`, and swap its `--config` for a copy of `litellm/proxy/dev_config.yaml` that points both `router_settings` (`redis_host`, `redis_port`) and `general_settings.coordination_redis` (`host`, `port`) at one dedicated Redis instance, with credentials when required, so the workers share router state and spend, budget, and rate-limit counters. Check that two worker PIDs are serving, and use the same worker count and Redis setup for the before and after proof. When the PR is about `--run_gunicorn`, `--run_hypercorn`, or `--run_granian`, which `--reload` does not support, drop `--reload`. When it is about behavior without Redis, drop Redis. When the proxy prints `"workers" flag is ignored when reloading is enabled`, that checkout or its uvicorn reloads only one worker, so drop the two-worker check for it. Say in the proof which part you dropped and why
+
 Simple PRs may skip before/after e2e when the full diff is mechanical and focused checks completely prove it, such as #49030 restoring an existing helper name. Trace affected callers and run relevant tests/static checks at the tip. For a bug fix, establish that the regression catches the original defect. Size alone never qualifies. New behavior, API/provider semantics, auth, billing, persistence, concurrency or uncertain integration risk need live e2e. In Screenshots / Proof of Fix, record `E2E exception at <tip SHA>`, scope, rationale, commands and results. Tests count as proof only for this exception. Reassess changed diffs, retain CI/bot gates, and honor explicit e2e requests (2026-10-28, unnecessary live QA for #49030)
 
 For other PRs, proof must show the end user's real flow before and after, with commands and output against a live proxy and real provider calls where applicable. Pytest alone is insufficient. For UI or interactive coding-tool flows, drive that surface and embed before/after screenshots, with URLs, clicks and input fields a reviewer can reproduce

README.md

@@ -696,7 +696,7 @@ For companies that need better security, user management and professional suppor
 [Talk to founders](https://enterprise.litellm.ai/demo)
 
 This covers:
-- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/proxy/enterprise):**
+- ✅ **Features under the [LiteLLM Commercial License](https://docs.litellm.ai/docs/enterprise):**
 - ✅ **Feature Prioritization**
 - ✅ **Custom Integrations**
 - ✅ **Professional Support - Dedicated discord + slack**

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -200,6 +200,7 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi
     unpriced: Final = _without_read_price(billed.response)
     cost: Final = logging_obj.response_cost_calculator(result=unpriced)
     end_time: Final = datetime.now()
+    logging_obj.capture_usage_before_logging(unpriced, logging_obj.start_time, end_time)
     GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
         async_coroutine=logging_obj.async_success_handler(
             result=unpriced, start_time=logging_obj.start_time, end_time=end_time

litellm-rust/crates/core-utils/src/core_helpers.rs

@@ -13,6 +13,7 @@ const FINISH_REASONS: &[(&str, &str)] = &[
     ("max_tokens", "length"),
     ("refusal", "content_filter"),
     ("compaction", "length"),
+    ("pause_turn", "length"),
     ("model_context_window_exceeded", "length"),
     ("guardrail_intervened", "content_filter"),
     ("content_filtered", "content_filter"),
@@ -75,6 +76,7 @@ mod tests {
     #[case::end_turn("end_turn", "stop")]
     #[case::stop_sequence("stop_sequence", "stop")]
     #[case::max_tokens("max_tokens", "length")]
+    #[case::pause_turn("pause_turn", "length")]
     #[case::model_context_window_exceeded("model_context_window_exceeded", "length")]
     #[case::refusal("refusal", "content_filter")]
     #[case::guardrail_intervened("guardrail_intervened", "content_filter")]

litellm-rust/crates/cost/AGENTS.md

@@ -0,0 +1 @@
+The price vocabulary (meter, band, channel, `When`, `Billing`) is explained in [../model-catalog/AGENTS.md](../model-catalog/AGENTS.md#concepts), the only crate that maps it to JSON keys

litellm-rust/crates/cost/src/price.rs

@@ -19,6 +19,7 @@ pub enum ServiceTier {
     Balanced,
 }
 
+/// A prompt-size range. `OverPromptTokens(n)` is spelled `_above_<n>_tokens` or `_above_<n / 1000>k_tokens`
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum Band {
     #[default]
@@ -26,12 +27,15 @@ pub enum Band {
     OverPromptTokens(NonZeroU64),
 }
 
+/// The request condition a price applies under, its service tier and prompt band
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct When {
     pub tier: ServiceTier,
     pub band: Band,
 }
 
+/// The route whose price list a call reads, from the `_batches` and `_decisions` suffixes. Not part of `When`,
+/// since a missing batch price is half the online rate and a missing decisions price is the full online rate
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum Channel {
     #[default]
@@ -40,6 +44,7 @@ pub enum Channel {
     Decisions,
 }
 
+/// The key of one rate, what is counted, through which channel and under which condition
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct Price {
     pub meter: Meter,
@@ -47,6 +52,7 @@ pub struct Price {
     pub when: When,
 }
 
+/// Every rate a row lists, its off-peak overlay and the row attributes that say how the rates apply
 #[derive(Clone, Debug, Default, PartialEq)]
 pub struct Prices {
     rates: BTreeMap<Price, UnitPrice>,
@@ -117,6 +123,7 @@ pub enum TierScope {
     WholeCall,
 }
 
+/// Rates that replace the row's token rates while a UTC window in the schedule is open
 #[derive(Clone, Debug, PartialEq)]
 pub struct OffPeak {
     schedule: Schedule,
@@ -133,6 +140,7 @@ pub struct OffPeakRates {
     pub reasoning: Option<UnitPrice>,
 }
 
+/// Multipliers applied on top of a priced call, never prices themselves
 #[derive(Clone, Debug, Default, PartialEq)]
 pub struct Adjustments(BTreeMap<Uplift, Multiplier>);
 

litellm-rust/crates/cost/src/price/meter.rs

@@ -1,5 +1,6 @@
 use std::num::NonZeroU64;
 
+/// What a price counts. A variation only one meter has, such as cache TTL or image size, lives inside its variant
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum Meter {
     Input(InputUnit),
@@ -51,12 +52,14 @@ pub enum OutputUnit {
     VideoSecond,
 }
 
+/// `Explicit` is a cache the caller created ahead of time, spelled `explicit_cache_read_*`. `Implicit` is every other read
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum CacheReadMode {
     Implicit,
     Explicit,
 }
 
+/// How long a cache write lives. `OneHour` and `TwelveHours` are spelled `_above_1hr` and `_above_12hr`
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum CacheTtl {
     Default,

litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs

@@ -124,12 +124,86 @@ pub enum AnthropicBeta {
     ToolSearchTool20251019,
     #[strum(serialize = "inline-tools-2026-09-15")]
     InlineTools20260915,
+    #[strum(serialize = "afk-mode-2026-01-31")]
+    AfkMode20260131,
+    #[strum(serialize = "agent-memory-2026-07-22")]
+    AgentMemory20260722,
+    #[strum(serialize = "analytics-usage-cost-2026-08-24")]
+    AnalyticsUsageCost20260824,
+    #[strum(serialize = "cache-diagnosis-2026-04-07")]
+    CacheDiagnosis20260407,
+    #[strum(serialize = "ce-plugins-2026-09-01")]
+    CePlugins20260901,
+    #[strum(serialize = "ce-user-management-2026-07-13")]
+    CeUserManagement20260713,
+    #[strum(serialize = "code-execution-2025-05-22")]
+    CodeExecution20250522,
+    #[strum(serialize = "computer-use-2024-10-22")]
+    ComputerUse20241022,
+    #[strum(serialize = "dev-full-thinking-2025-05-14")]
+    DevFullThinking20250514,
+    #[strum(serialize = "dreaming-2026-04-21")]
+    Dreaming20260421,
+    #[strum(serialize = "experimental-claude-tag-2026-10-21")]
+    ExperimentalClaudeTag20261021,
+    #[strum(serialize = "extended-cache-ttl-2025-04-11")]
+    ExtendedCacheTtl20250411,
+    #[strum(serialize = "fallback-credit-2026-06-01")]
+    FallbackCredit20260601,
+    #[strum(serialize = "fallback-credit-2026-07-01")]
+    FallbackCredit20260701,
+    #[strum(serialize = "managed-agents-2026-04-01")]
+    ManagedAgents20260401,
+    #[strum(serialize = "mcp-client-2026-09-15")]
+    McpClient20260915,
+    #[strum(serialize = "mcp-tunnels-2026-06-22")]
+    McpTunnels20260622,
+    #[strum(serialize = "message-batches-2024-09-24")]
+    MessageBatches20240924,
+    #[strum(serialize = "message-threads-2026-08-12")]
+    MessageThreads20260812,
+    #[strum(serialize = "mid-conversation-system-clear-at-2026-08-21")]
+    MidConversationSystemClearAt20260821,
+    #[strum(serialize = "model-context-window-exceeded-2025-08-26")]
+    ModelContextWindowExceeded20250826,
+    #[strum(serialize = "output-300k-2026-03-24")]
+    Output300k20260324,
+    #[strum(serialize = "pdfs-2024-09-25")]
+    Pdfs20240925,
+    #[strum(serialize = "prompt-caching-2024-07-31")]
+    PromptCaching20240731,
+    #[strum(serialize = "redact-thinking-2026-02-12")]
+    RedactThinking20260212,
+    #[strum(serialize = "server-side-fallback-2026-06-01")]
+    ServerSideFallback20260601,
+    #[strum(serialize = "server-side-fallback-2026-07-01")]
+    ServerSideFallback20260701,
+    #[strum(serialize = "spend-limit-reads-2026-09-26")]
+    SpendLimitReads20260926,
+    #[strum(serialize = "structured-outputs-2025-12-15")]
+    StructuredOutputs20251215,
+    #[strum(serialize = "task-budgets-2026-03-13")]
+    TaskBudgets20260313,
+    #[strum(serialize = "telemetry-destinations-2026-08-11")]
+    TelemetryDestinations20260811,
+    #[strum(serialize = "thinking-token-count-2026-05-13")]
+    ThinkingTokenCount20260513,
+    #[strum(serialize = "token-counting-2024-11-01")]
+    TokenCounting20241101,
+    #[strum(serialize = "user-profiles-2026-03-24")]
+    UserProfiles20260324,
+    #[strum(serialize = "user-profiles-2026-08-18")]
+    UserProfiles20260818,
+    #[strum(serialize = "user-profiles-2026-09-04")]
+    UserProfiles20260904,
+    #[strum(serialize = "webhooks-2026-10-07")]
+    Webhooks20261007,
     #[strum(default, transparent)]
     Other(String),
 }
 
 impl AnthropicBeta {
-    pub const KNOWN: [Self; 40] = [
+    pub const KNOWN: [Self; 77] = [
         Self::Oauth20250420,
         Self::WebFetch20250910,
         Self::WebSearch20250305,
@@ -170,6 +244,43 @@ impl AnthropicBeta {
         Self::ToolExamples20251029,
         Self::ToolSearchTool20251019,
         Self::InlineTools20260915,
+        Self::AfkMode20260131,
+        Self::AgentMemory20260722,
+        Self::AnalyticsUsageCost20260824,
+        Self::CacheDiagnosis20260407,
+        Self::CePlugins20260901,
+        Self::CeUserManagement20260713,
+        Self::CodeExecution20250522,
+        Self::ComputerUse20241022,
+        Self::DevFullThinking20250514,
+        Self::Dreaming20260421,
+        Self::ExperimentalClaudeTag20261021,
+        Self::ExtendedCacheTtl20250411,
+        Self::FallbackCredit20260601,
+        Self::FallbackCredit20260701,
+        Self::ManagedAgents20260401,
+        Self::McpClient20260915,
+        Self::McpTunnels20260622,
+        Self::MessageBatches20240924,
+        Self::MessageThreads20260812,
+        Self::MidConversationSystemClearAt20260821,
+        Self::ModelContextWindowExceeded20250826,
+        Self::Output300k20260324,
+        Self::Pdfs20240925,
+        Self::PromptCaching20240731,
+        Self::RedactThinking20260212,
+        Self::ServerSideFallback20260601,
+        Self::ServerSideFallback20260701,
+        Self::SpendLimitReads20260926,
+        Self::StructuredOutputs20251215,
+        Self::TaskBudgets20260313,
+        Self::TelemetryDestinations20260811,
+        Self::ThinkingTokenCount20260513,
+        Self::TokenCounting20241101,
+        Self::UserProfiles20260324,
+        Self::UserProfiles20260818,
+        Self::UserProfiles20260904,
+        Self::Webhooks20261007,
     ];
 
     pub fn as_str(&self) -> &str {
@@ -195,10 +306,9 @@ impl AnthropicBeta {
 
     fn rejected_by(&self) -> &'static [BetaProvider] {
         match self {
-            Self::AdvancedToolUse20251120 => {
+            Self::AdvancedToolUse20251120 | Self::ToolSearchTool20251019 => {
                 &[BetaProvider::BedrockConverse, BetaProvider::Databricks]
             }
-            Self::ContextManagement20250627 => &[BetaProvider::BedrockConverse],
             Self::AdvisorTool20260301 => &[
                 BetaProvider::AzureAi,
                 BetaProvider::BedrockConverse,
@@ -207,98 +317,99 @@ impl AnthropicBeta {
                 BetaProvider::VertexAi,
                 BetaProvider::Databricks,
             ],
-            Self::Compact20260904 => &[
+            Self::AfkMode20260131
+            | Self::CePlugins20260901
+            | Self::CeUserManagement20260713
+            | Self::Compact20260904
+            | Self::Dreaming20260421
+            | Self::ExperimentalClaudeTag20261021
+            | Self::ExtendedCacheTtl20250411
+            | Self::FallbackCredit20260601
+            | Self::FallbackCredit20260701
+            | Self::InlineTools20260915
+            | Self::ManagedAgents20260401
+            | Self::McpTunnels20260622
+            | Self::MidConversationSystemClearAt20260821
+            | Self::ModelContextWindowExceeded20250826
+            | Self::Oauth20250420
+            | Self::Pdfs20240925
+            | Self::PromptCaching20240731
+            | Self::PromptCachingScope20260105
+            | Self::RedactThinking20260212
+            | Self::ServerSideFallback20260601
+            | Self::SpendLimitReads20260926
+            | Self::StructuredOutputs20251215
+            | Self::TaskBudgets20260313
+            | Self::TelemetryDestinations20260811
+            | Self::ThinkingTokenCount20260513
+            | Self::TokenCounting20241101
+            | Self::UserProfiles20260324
+            | Self::UserProfiles20260818
+            | Self::UserProfiles20260904 => &[
                 BetaProvider::BedrockConverse,
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::Databricks,
             ],
-            Self::Bash20241022
+            Self::AgentMemory20260722
+            | Self::Bash20241022
             | Self::Bash20250124
+            | Self::DevFullThinking20250514
             | Self::McpServers20251204
             | Self::StructuredOutput20240301
             | Self::TextEditor20241022
             | Self::TextEditor20250124 => BetaProvider::VARIANTS,
-            Self::ClaudeCode20250219 | Self::ToolExamples20251029 => &[
-                BetaProvider::Anthropic,
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::VertexAi,
-                BetaProvider::Databricks,
-            ],
-            Self::CodeExecution20250825
+            Self::AnalyticsUsageCost20260824
+            | Self::CacheDiagnosis20260407
+            | Self::CodeExecution20250522
+            | Self::CodeExecution20250825
+            | Self::ComputerUse20241022
+            | Self::FastMode20260201
             | Self::FilesApi20250414
             | Self::McpClient20250404
             | Self::McpClient20251120
-            | Self::PromptCachingScope20260105
+            | Self::McpClient20260915
+            | Self::MessageBatches20240924
+            | Self::MessageThreads20260812
+            | Self::Output300k20260324
+            | Self::ServerSideFallback20260701
             | Self::Skills20251002
-            | Self::WebFetch20250910 => &[
+            | Self::WebFetch20250910
+            | Self::Webhooks20261007 => &[
                 BetaProvider::BedrockConverse,
                 BetaProvider::Bedrock,
                 BetaProvider::BedrockMantle,
                 BetaProvider::VertexAi,
                 BetaProvider::Databricks,
             ],
-            Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
+            Self::ClaudeCode20250219
+            | Self::PerTurnControl20260701
+            | Self::ToolExamples20251029 => &[
+                BetaProvider::BedrockConverse,
+                BetaProvider::Bedrock,
+                BetaProvider::Databricks,
+            ],
+            Self::Compact20260112
+            | Self::ContextManagement20250627
+            | Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
             Self::ComputerUse20250124
             | Self::ComputerUse20251124
             | Self::Context1m20250807
+            | Self::Effort20251124
+            | Self::FineGrainedToolStreaming20250514
             | Self::ThinkingBindingControls20260801
             | Self::ThinkingDisplayUpdates20260818 => &[],
-            Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
-            Self::Effort20251124 => &[BetaProvider::VertexAi],
-            Self::FastMode20260201 | Self::Oauth20250420 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::BedrockMantle,
-                BetaProvider::VertexAi,
-                BetaProvider::Databricks,
-            ],
-            Self::FineGrainedToolStreaming20250514 => {
-                &[BetaProvider::AzureAi, BetaProvider::VertexAi]
-            }
-            Self::InterleavedThinking20250514 | Self::WebSearch20250305 => {
-                &[BetaProvider::BedrockConverse, BetaProvider::Bedrock]
-            }
-            Self::MidConversationOutputConfig20260701 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::BedrockMantle,
-                BetaProvider::VertexAi,
-                BetaProvider::Databricks,
-            ],
-            Self::MidConversationToolChanges20260701 => &[
+            Self::InterleavedThinking20250514
+            | Self::Output128k20250219
+            | Self::TokenEfficientTools20250219
+            | Self::WebSearch20250305 => &[BetaProvider::BedrockConverse, BetaProvider::Bedrock],
+            Self::MidConversationOutputConfig20260701
+            | Self::MidConversationToolChanges20260701 => &[
                 BetaProvider::BedrockConverse,
                 BetaProvider::BedrockMantle,
                 BetaProvider::Databricks,
             ],
-            Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::VertexAi,
-            ],
-            Self::PerTurnControl20260701 => &[
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::Databricks,
-            ],
-            Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
-            Self::ToolSearchTool20251019 => &[
-                BetaProvider::Anthropic,
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Databricks,
-            ],
-            Self::InlineTools20260915 => &[
-                BetaProvider::AzureAi,
-                BetaProvider::BedrockConverse,
-                BetaProvider::Bedrock,
-                BetaProvider::BedrockMantle,
-                BetaProvider::Databricks,
-            ],
+            Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock],
             Self::Other(_) => BetaProvider::VARIANTS,
         }
     }
@@ -451,7 +562,44 @@ mod tests {
             AnthropicBeta::TokenEfficientTools20250219,
             AnthropicBeta::ToolExamples20251029,
             AnthropicBeta::ToolSearchTool20251019,
-            AnthropicBeta::InlineTools20260915
+            AnthropicBeta::InlineTools20260915,
+            AnthropicBeta::AfkMode20260131,
+            AnthropicBeta::AgentMemory20260722,
+            AnthropicBeta::AnalyticsUsageCost20260824,
+            AnthropicBeta::CacheDiagnosis20260407,
+            AnthropicBeta::CePlugins20260901,
+            AnthropicBeta::CeUserManagement20260713,
+            AnthropicBeta::CodeExecution20250522,
+            AnthropicBeta::ComputerUse20241022,
+            AnthropicBeta::DevFullThinking20250514,
+            AnthropicBeta::Dreaming20260421,
+            AnthropicBeta::ExperimentalClaudeTag20261021,
+            AnthropicBeta::ExtendedCacheTtl20250411,
+            AnthropicBeta::FallbackCredit20260601,
+            AnthropicBeta::FallbackCredit20260701,
+            AnthropicBeta::ManagedAgents20260401,
+            AnthropicBeta::McpClient20260915,
+            AnthropicBeta::McpTunnels20260622,
+            AnthropicBeta::MessageBatches20240924,
+            AnthropicBeta::MessageThreads20260812,
+            AnthropicBeta::MidConversationSystemClearAt20260821,
+            AnthropicBeta::ModelContextWindowExceeded20250826,
+            AnthropicBeta::Output300k20260324,
+            AnthropicBeta::Pdfs20240925,
+            AnthropicBeta::PromptCaching20240731,
+            AnthropicBeta::RedactThinking20260212,
+            AnthropicBeta::ServerSideFallback20260601,
+            AnthropicBeta::ServerSideFallback20260701,
+            AnthropicBeta::SpendLimitReads20260926,
+            AnthropicBeta::StructuredOutputs20251215,
+            AnthropicBeta::TaskBudgets20260313,
+            AnthropicBeta::TelemetryDestinations20260811,
+            AnthropicBeta::ThinkingTokenCount20260513,
+            AnthropicBeta::TokenCounting20241101,
+            AnthropicBeta::UserProfiles20260324,
+            AnthropicBeta::UserProfiles20260818,
+            AnthropicBeta::UserProfiles20260904,
+            AnthropicBeta::Webhooks20261007
         )]
         beta: AnthropicBeta,
     ) {
@@ -551,7 +699,7 @@ mod tests {
         BetaProvider::Anthropic,
         Some(AnthropicBeta::Oauth20250420)
     )]
-    #[case::rejected("effort-2025-11-24", BetaProvider::VertexAi, None)]
+    #[case::rejected("skills-2025-10-02", BetaProvider::VertexAi, None)]
     #[case::unknown("example-beta-2099-01-01", BetaProvider::Anthropic, None)]
     fn on_resolves_known_spellings_held_as_other(
         #[case] raw: &str,
@@ -580,12 +728,11 @@ mod tests {
 
     #[test]
     fn unknown_values_are_kept_verbatim() {
-        let parsed: AnthropicBeta = "claude-code-20250219".parse().unwrap();
-        assert_eq!(
-            parsed,
-            AnthropicBeta::Other("claude-code-20250219".to_string())
+        let parsed: AnthropicBeta = "example-beta-2099-01-01".parse().unwrap();
+        assert!(
+            matches!(&parsed, AnthropicBeta::Other(value) if value == "example-beta-2099-01-01")
         );
-        assert_eq!(parsed.to_string(), "claude-code-20250219");
+        assert_eq!(parsed.to_string(), "example-beta-2099-01-01");
     }
 
     #[test]

litellm-rust/crates/llms/src/anthropic/beta_headers.rs

@@ -96,7 +96,10 @@ mod tests {
     }
 
     #[rstest]
-    #[case::azure_ai(BetaProvider::AzureAi, "fast-mode-2026-02-01,example-beta-2099-01-01")]
+    #[case::azure_ai(
+        BetaProvider::AzureAi,
+        "advisor-tool-2026-03-01,example-beta-2099-01-01"
+    )]
     #[case::anthropic(BetaProvider::Anthropic, "bash_20241022")]
     #[case::vertex_ai(BetaProvider::VertexAi, " , ")]
     fn filter_removes_header_when_nothing_survives(

litellm-rust/crates/model-catalog/AGENTS.md

@@ -1,3 +1,41 @@
+## Concepts
+
+Every price in a row lowers to one `Price { meter, channel, when }` and its `UnitPrice`. These keys come from the `FlatKey::parse` tests
+
+- `input_cost_per_token`
+  - meter `Input(Token(Text))`, channel `Online`, tier `Standard`, band `Base`
+- `output_cost_per_token_above_272k_tokens_priority`
+  - meter `Output(Token(Text))`, channel `Online`
+  - `_priority` sets tier `Priority`
+  - `_above_272k_tokens` sets band `OverPromptTokens(272000)`
+- `cache_creation_input_token_cost_above_1hr_above_200k_tokens`
+  - meter `CacheWrite { Text, OneHour }`, where `_above_1hr` is the cache TTL, not a band
+  - `_above_200k_tokens` sets band `OverPromptTokens(200000)`
+  - channel `Online`, tier `Standard`
+- `ocr_cost_per_page_batches`
+  - meter `Feature(OcrPage)`, tier `Standard`, band `Base`
+  - `_batches` sets channel `Batch`
+
+- Meter is what gets counted, such as text input tokens, reasoning tokens, an image of a given size or a web search
+  - A variation only one meter has stays inside it (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`)
+- `UnitPrice` is USD for one unit of the meter, such as one token, image, page or second
+- Band is a range of prompt sizes, starting where `_above_<N>_tokens` or `_above_<N>k_tokens` says
+  - `Prices::band` picks one band from prompt tokens alone, the highest edge passed wins, and `Band::Base` is below every edge
+  - The whole call moves, so every token meter, output included, bills all its tokens at that band's rate
+  - `Billing::band_edge` says whether a prompt exactly at the edge is in the band (exclusive by default, inclusive for xAI)
+  - A band only another tier or the batch channel prices never moves the call
+- `When` is the request's condition, its service tier and band
+  - A missing price falls back from `(tier, band)` to `(standard, band)`, `(tier, base)`, then `(standard, base)`
+- `Channel` is the route whose price list the call reads, and each falls back differently
+  - `Online` falls back across `When` as above
+  - `Batch` (`_batches`) uses bands but no service tier, and a missing price is half the online rate
+  - `Decisions` (`_decisions`, OpenAI `/v1/decisions`) overrides single online fields, and a missing price is the full online rate
+- Off-peak and `Adjustments` are never prices
+  - Off-peak swaps in other token rates while a UTC window is open
+  - `Adjustments` multiplies a priced call for region, residency, geo or `speed: fast`
+- `Billing` on `Prices` holds row attributes that change how its prices apply (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope)
+  - `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list
+
 ## Layout
 
 `wire/` reads one entry as written. `EntryWire::read` makes one pass over the entry's keys and puts every non-null key in exactly one place, which is an attribute, a flat price parsed by `FlatKey::parse`, a price block, a `supports_*` capability or `unknown`. A known key with a bad value becomes an `Error` that names the key, and the rest of the entry is still read
@@ -6,8 +44,6 @@
 
 `catalog.rs` reads the document, indexes aliases and runs the checks across entries. A bad entry keeps its raw fields and its errors are listed in `Loaded.errors`. Deciding what to do with them is the caller's job. `Catalog::written` gives back every top-level key and value as the document wrote them, in document order. `Catalog::served` gives back every model and alias as Python serves it, without `sample_spec` and `fallback_generalizations`
 
-`Billing` on `Prices` holds the row attributes pricing reads (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope). `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list
-
 `store/` is the catalog's in-memory store and the only place that indexes model names. Its rules are in [src/store/AGENTS.md](src/store/AGENTS.md)
 
 `layers.rs`, `resolve/`, `model_info.rs` and `register.rs` hold types only. The functions over them land with their first caller
@@ -16,7 +52,7 @@ Catalog errors use one opaque `Error` with typed causes and JSON Pointer accesso
 
 ## Prices
 
-The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names. Every price is keyed by `Meter`, `Channel` and `When`. A variation only one meter has stays in the meter (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`). Service tier and prompt band make up `When`. Batch and decisions are a `Channel` because Python bills them with their own rules. Off-peak is an overlay on `Prices`, and regional uplifts are `Adjustments`, never prices
+The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names
 
 A new `N`, image size or resolution in an existing key shape parses with no Rust change. A new unit, token kind, feature or tier needs a variant in `litellm-cost` plus its spelling in `wire/flat_key.rs` and `FLAT_KEY_PATTERN` in `schema.rs`
 

litellm-rust/crates/model-catalog/src/wire/attributes.rs

@@ -11,7 +11,7 @@ use crate::error::{DateCause, Error, Repr};
 use crate::vocabulary::{
     AudioFormat, EffortLevel, Endpoint, InputModality, Mode, OutputModality, VertexAiAudioApi,
 };
-use crate::wire::lax::Count;
+use crate::wire::lax::{Count, NumericCount};
 
 macro_rules! attributes {
     ($($(#[$doc:meta])* $name:ident: $ty:ty $(as $via:ty)?),* $(,)?) => {
@@ -104,9 +104,12 @@ attributes! {
     /// Cloud regions the model is available in ('global' or region ids).
     supported_regions: Vec<String>,
     thinking_always_on: bool,
-    token_count_framing: u64 as Count,
+    token_count_framing: u64 as NumericCount,
     token_count_model: TokenCountModel,
+    token_count_responses_framing: u64 as NumericCount,
+    token_count_responses_tool_framing: u64 as NumericCount,
     token_count_tokenizer: TokenCountTokenizer,
+    token_count_tool_framing: u64 as NumericCount,
     /// Provider default tokens-per-minute limit.
     tpm: u64 as Count,
     use_openai_responses_path: bool,
@@ -302,6 +305,18 @@ mod tests {
     #[case::reasoning_modes("reasoning_modes", json!(["none", "adaptive"]), true)]
     #[case::reasoning_context_not_a_list("reasoning_contexts", json!("reasoning"), false)]
     #[case::reasoning_mode_not_a_string("reasoning_modes", json!([1]), false)]
+    #[case::framing("token_count_framing", json!(12), true)]
+    #[case::framing_written_as_text("token_count_framing", json!("12"), false)]
+    #[case::framing_written_as_whole_float("token_count_framing", json!(12.0), true)]
+    #[case::fractional_tool_framing("token_count_tool_framing", json!(1.5), false)]
+    #[case::tool_framing("token_count_tool_framing", json!(214), true)]
+    #[case::tool_framing_written_as_text("token_count_tool_framing", json!("214"), false)]
+    #[case::tool_framing_as_a_flag("token_count_tool_framing", json!(true), false)]
+    #[case::negative_tool_framing("token_count_tool_framing", json!(-1), false)]
+    #[case::responses_framing("token_count_responses_framing", json!(38), true)]
+    #[case::responses_framing_as_a_flag("token_count_responses_framing", json!(false), false)]
+    #[case::responses_tool_framing("token_count_responses_tool_framing", json!(267), true)]
+    #[case::responses_tool_framing_written_as_text("token_count_responses_tool_framing", json!("267"), false)]
     fn each_attribute_reads_its_own_type(
         #[case] key: &str,
         #[case] value: Value,

litellm-rust/crates/model-catalog/src/wire/lax.rs

@@ -8,6 +8,8 @@ pub(crate) struct LaxI64;
 
 pub(crate) struct Count;
 
+pub(crate) struct NumericCount;
+
 impl<'de> DeserializeAs<'de, i64> for LaxI64 {
     fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<i64, D::Error> {
         deserializer.deserialize_any(Self)
@@ -52,6 +54,34 @@ impl<'de> DeserializeAs<'de, u64> for Count {
     }
 }
 
+impl<'de> DeserializeAs<'de, u64> for NumericCount {
+    fn deserialize_as<D: Deserializer<'de>>(deserializer: D) -> Result<u64, D::Error> {
+        deserializer.deserialize_any(Self)
+    }
+}
+
+impl Visitor<'_> for NumericCount {
+    type Value = u64;
+
+    fn expecting(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        formatter.write_str("a whole number of zero or more")
+    }
+
+    fn visit_u64<E: de::Error>(self, value: u64) -> Result<u64, E> {
+        Ok(value)
+    }
+
+    fn visit_i64<E: de::Error>(self, value: i64) -> Result<u64, E> {
+        u64::try_from(value).map_err(|_| E::invalid_value(de::Unexpected::Signed(value), &self))
+    }
+
+    fn visit_f64<E: de::Error>(self, value: f64) -> Result<u64, E> {
+        integral_float(value)
+            .and_then(|whole| u64::try_from(whole).ok())
+            .ok_or_else(|| E::invalid_value(de::Unexpected::Float(value), &self))
+    }
+}
+
 fn integer_string(value: &str) -> Option<i64> {
     let integer = match value.split_once('.') {
         Some((integer, fraction)) => {
@@ -99,6 +129,12 @@ mod tests {
             .map(DeserializeAsWrap::into_inner)
     }
 
+    fn numeric_count(value: Value) -> Option<u64> {
+        DeserializeAsWrap::<u64, NumericCount>::deserialize(value)
+            .ok()
+            .map(DeserializeAsWrap::into_inner)
+    }
+
     use serde::Deserialize;
 
     #[rstest]
@@ -125,4 +161,23 @@ mod tests {
     ) {
         assert_eq!(count(written), expected);
     }
+
+    #[rstest]
+    #[case::integer(json!(214), Some(214))]
+    #[case::zero(json!(0), Some(0))]
+    #[case::whole_float(json!(214.0), Some(214))]
+    #[case::negative_zero(json!(-0.0), Some(0))]
+    #[case::above_i64(json!(u64::MAX), Some(u64::MAX))]
+    #[case::negative(json!(-1), None)]
+    #[case::negative_whole_float(json!(-1.0), None)]
+    #[case::fraction(json!(1.5), None)]
+    #[case::string(json!("214"), None)]
+    #[case::boolean(json!(true), None)]
+    #[case::list(json!([214]), None)]
+    fn numeric_counts_accept_only_whole_json_numbers(
+        #[case] written: Value,
+        #[case] expected: Option<u64>,
+    ) {
+        assert_eq!(numeric_count(written), expected);
+    }
 }

litellm/__init__.py

@@ -1647,6 +1647,7 @@ def set_global_gitlab_config(config: Mapping[str, object]) -> None:
 # Lazy loading system for heavy modules to reduce initial import time and memory usage
 
 if TYPE_CHECKING:
+    from . import guardrails as guardrails
     from litellm.types.utils import ModelInfo as _ModelInfoType
     from litellm.types.utils import PriorityReservationSettings
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
@@ -2499,10 +2500,10 @@ def __getattr__(name: str) -> Any:
         handler_func: Final = registry[name]
         return handler_func(name)
 
-    if name == "proxy":
+    if name in ("proxy", "guardrails"):
         import importlib
 
-        return importlib.import_module("litellm.proxy")
+        return importlib.import_module(f"litellm.{name}")
 
     if name == "harness" or name in _AGENT_EXPORTS:
         import importlib

litellm/_internal_context.py

@@ -21,6 +21,7 @@ _T = TypeVar("_T")
 # When True, suppresses async logging and billing for internal sub-calls
 # (e.g., emulated file-search steps that make nested LLM calls).
 is_internal_call: Final[ContextVar[bool]] = ContextVar("is_internal_call", default=False)
+internal_parent_request_id: Final[ContextVar[str | None]] = ContextVar("internal_parent_request_id", default=None)
 
 # One request prices its totals, its per-token-type lines and the rates it reports on
 # separate code paths. Each reads the clock for off-peak pricing, so without a pinned
@@ -40,13 +41,15 @@ _mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=Fa
 
 
 @contextmanager
-def internal_sub_call() -> Generator[None]:
+def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]:
     """A nested call the parent call logs and bills, so the parent must fold this call's cost into its own."""
     token: Final = is_internal_call.set(True)
+    parent_token: Final = internal_parent_request_id.set(parent_request_id or internal_parent_request_id.get())
     try:
         yield
     finally:
         is_internal_call.reset(token)
+        internal_parent_request_id.reset(parent_token)
 
 
 @contextmanager

litellm/accounting/protocols.py

@@ -3,7 +3,7 @@
 The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
 Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
 model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
-These inputs must survive request teardown. Cost-map rates are resolved at compilation, not capture.
+These inputs, including the effective cost-map rates, must survive request teardown.
 Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
 T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
 """
@@ -38,8 +38,8 @@ class CostCompiler(Protocol):
     async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
         """Reuse the committed charge or finalize one charge and its delivery intent.
 
-        Resolve captured request-scoped context by event identity and cost-map rates
-        at compilation. Persist the resolved inputs in the required snapshot. Replay
+        Resolve captured request-scoped context and rates by event identity.
+        Persist the resolved inputs in the required snapshot. Replay
         reuses the committed result and never consumes free allowance again.
         Missing pricing is Unpriced, never an inferred zero.
         """

litellm/accounting/types.py

@@ -276,8 +276,8 @@ class NamedCost(BaseModel):
 class PricingSnapshot(BaseModel):
     """Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
 
-    Request-scoped inputs and overrides come from the usage row's versioned payload. Cost-map rates are
-    resolved at compilation, not frozen at capture. Retries reuse the committed charge and snapshot.
+    Request-scoped inputs, overrides and effective rates come from the usage row's versioned payload.
+    Retries reuse the committed charge and snapshot.
     Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
     Component sums and float tolerances belong to reconciliation, not this structural contract.
     """

litellm/anthropic_beta_headers_config.json

@@ -37,7 +37,47 @@
     "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
     "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
     "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "agent-memory-2026-07-22": null,
+    "analytics-usage-cost-2026-08-24": "analytics-usage-cost-2026-08-24",
+    "cache-diagnosis-2026-04-07": "cache-diagnosis-2026-04-07",
+    "ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
+    "ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
+    "claude-code-20250219": "claude-code-20250219",
+    "code-execution-2025-05-22": "code-execution-2025-05-22",
+    "computer-use-2024-10-22": "computer-use-2024-10-22",
+    "dev-full-thinking-2025-05-14": null,
+    "dreaming-2026-04-21": "dreaming-2026-04-21",
+    "experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
+    "extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
+    "fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
+    "fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
+    "managed-agents-2026-04-01": "managed-agents-2026-04-01",
+    "mcp-client-2026-09-15": "mcp-client-2026-09-15",
+    "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
+    "message-batches-2024-09-24": "message-batches-2024-09-24",
+    "message-threads-2026-08-12": "message-threads-2026-08-12",
+    "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
+    "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
+    "output-300k-2026-03-24": "output-300k-2026-03-24",
+    "pdfs-2024-09-25": "pdfs-2024-09-25",
+    "prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
+    "server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
+    "server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01",
+    "spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
+    "task-budgets-2026-03-13": "task-budgets-2026-03-13",
+    "telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
+    "thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
+    "token-counting-2024-11-01": "token-counting-2024-11-01",
+    "tool-examples-2025-10-29": "tool-examples-2025-10-29",
+    "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
+    "user-profiles-2026-03-24": "user-profiles-2026-03-24",
+    "user-profiles-2026-08-18": "user-profiles-2026-08-18",
+    "user-profiles-2026-09-04": "user-profiles-2026-09-04",
+    "webhooks-2026-10-07": "webhooks-2026-10-07",
+    "afk-mode-2026-01-31": "afk-mode-2026-01-31",
+    "redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
+    "structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
   },
   "azure_ai": {
     "advisor-tool-2026-03-01": null,
@@ -45,23 +85,21 @@
     "bash_20241022": null,
     "bash_20250124": null,
     "code-execution-2025-08-25": "code-execution-2025-08-25",
-    "compact-2026-01-12": null,
-    "compact-2026-09-04": "compact-2026-09-04",
+    "compact-2026-01-12": "compact-2026-01-12",
     "computer-use-2025-01-24": "computer-use-2025-01-24",
     "computer-use-2025-11-24": "computer-use-2025-11-24",
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
     "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
     "effort-2025-11-24": "effort-2025-11-24",
-    "fast-mode-2026-02-01": null,
+    "fast-mode-2026-02-01": "fast-mode-2026-02-01",
     "files-api-2025-04-14": "files-api-2025-04-14",
-    "fine-grained-tool-streaming-2025-05-14": null,
+    "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
     "mcp-client-2025-11-20": "mcp-client-2025-11-20",
     "mcp-client-2025-04-04": "mcp-client-2025-04-04",
     "mcp-servers-2025-12-04": null,
-    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "output-128k-2025-02-19": null,
+    "output-128k-2025-02-19": "output-128k-2025-02-19",
     "structured-output-2024-03-01": null,
     "per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
     "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
@@ -71,9 +109,54 @@
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
     "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
-    "token-efficient-tools-2025-02-19": null,
+    "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
     "web-fetch-2025-09-10": "web-fetch-2025-09-10",
-    "web-search-2025-03-05": "web-search-2025-03-05"
+    "web-search-2025-03-05": "web-search-2025-03-05",
+    "agent-memory-2026-07-22": null,
+    "analytics-usage-cost-2026-08-24": "analytics-usage-cost-2026-08-24",
+    "cache-diagnosis-2026-04-07": "cache-diagnosis-2026-04-07",
+    "ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
+    "ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
+    "claude-code-20250219": "claude-code-20250219",
+    "code-execution-2025-05-22": "code-execution-2025-05-22",
+    "compact-2026-09-04": "compact-2026-09-04",
+    "computer-use-2024-10-22": "computer-use-2024-10-22",
+    "dev-full-thinking-2025-05-14": null,
+    "dreaming-2026-04-21": "dreaming-2026-04-21",
+    "experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
+    "extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
+    "fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
+    "fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "managed-agents-2026-04-01": "managed-agents-2026-04-01",
+    "mcp-client-2026-09-15": "mcp-client-2026-09-15",
+    "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
+    "message-batches-2024-09-24": "message-batches-2024-09-24",
+    "message-threads-2026-08-12": "message-threads-2026-08-12",
+    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
+    "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
+    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
+    "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
+    "oauth-2025-04-20": "oauth-2025-04-20",
+    "output-300k-2026-03-24": "output-300k-2026-03-24",
+    "pdfs-2024-09-25": "pdfs-2024-09-25",
+    "prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
+    "server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
+    "server-side-fallback-2026-07-01": "server-side-fallback-2026-07-01",
+    "spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
+    "task-budgets-2026-03-13": "task-budgets-2026-03-13",
+    "telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
+    "thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
+    "token-counting-2024-11-01": "token-counting-2024-11-01",
+    "tool-examples-2025-10-29": "tool-examples-2025-10-29",
+    "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
+    "user-profiles-2026-03-24": "user-profiles-2026-03-24",
+    "user-profiles-2026-08-18": "user-profiles-2026-08-18",
+    "user-profiles-2026-09-04": "user-profiles-2026-09-04",
+    "webhooks-2026-10-07": "webhooks-2026-10-07",
+    "afk-mode-2026-01-31": "afk-mode-2026-01-31",
+    "redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
+    "structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
   },
   "bedrock_converse": {
     "advisor-tool-2026-03-01": null,
@@ -196,34 +279,92 @@
     "context-1m-2025-08-07": "context-1m-2025-08-07",
     "context-management-2025-06-27": "context-management-2025-06-27",
     "dangerous-tool-use-2026-09-03": "dangerous-tool-use-2026-09-03",
-    "effort-2025-11-24": null,
+    "effort-2025-11-24": "effort-2025-11-24",
     "fast-mode-2026-02-01": null,
     "files-api-2025-04-14": null,
-    "fine-grained-tool-streaming-2025-05-14": null,
+    "fine-grained-tool-streaming-2025-05-14": "fine-grained-tool-streaming-2025-05-14",
     "interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
     "mcp-client-2025-11-20": null,
     "mcp-client-2025-04-04": null,
     "mcp-servers-2025-12-04": null,
-    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
-    "output-128k-2025-02-19": null,
+    "output-128k-2025-02-19": "output-128k-2025-02-19",
     "structured-output-2024-03-01": null,
     "per-turn-control-2026-07-01": "per-turn-control-2026-07-01",
     "prompt-caching-scope-2026-01-05": null,
     "skills-2025-10-02": null,
-    "structured-outputs-2025-11-13": null,
+    "structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
     "text_editor_20241022": null,
     "text_editor_20250124": null,
     "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
     "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
-    "token-efficient-tools-2025-02-19": null,
+    "token-efficient-tools-2025-02-19": "token-efficient-tools-2025-02-19",
     "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19",
     "web-fetch-2025-09-10": null,
     "web-search-2025-03-05": "web-search-2025-03-05",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "agent-memory-2026-07-22": null,
+    "analytics-usage-cost-2026-08-24": null,
+    "cache-diagnosis-2026-04-07": null,
+    "ce-plugins-2026-09-01": null,
+    "ce-user-management-2026-07-13": null,
+    "claude-code-20250219": "claude-code-20250219",
+    "code-execution-2025-05-22": null,
+    "compact-2026-09-04": null,
+    "computer-use-2024-10-22": null,
+    "dev-full-thinking-2025-05-14": null,
+    "dreaming-2026-04-21": null,
+    "experimental-claude-tag-2026-10-21": null,
+    "extended-cache-ttl-2025-04-11": null,
+    "fallback-credit-2026-06-01": "fallback-credit-2026-06-01",
+    "fallback-credit-2026-07-01": "fallback-credit-2026-07-01",
+    "managed-agents-2026-04-01": null,
+    "mcp-client-2026-09-15": null,
+    "mcp-tunnels-2026-06-22": null,
+    "message-batches-2024-09-24": null,
+    "message-threads-2026-08-12": null,
+    "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01",
+    "mid-conversation-system-clear-at-2026-08-21": "mid-conversation-system-clear-at-2026-08-21",
+    "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01",
+    "model-context-window-exceeded-2025-08-26": "model-context-window-exceeded-2025-08-26",
+    "oauth-2025-04-20": null,
+    "output-300k-2026-03-24": null,
+    "pdfs-2024-09-25": "pdfs-2024-09-25",
+    "prompt-caching-2024-07-31": null,
+    "server-side-fallback-2026-06-01": "server-side-fallback-2026-06-01",
+    "server-side-fallback-2026-07-01": null,
+    "spend-limit-reads-2026-09-26": null,
+    "task-budgets-2026-03-13": "task-budgets-2026-03-13",
+    "telemetry-destinations-2026-08-11": null,
+    "thinking-token-count-2026-05-13": "thinking-token-count-2026-05-13",
+    "token-counting-2024-11-01": "token-counting-2024-11-01",
+    "tool-examples-2025-10-29": "tool-examples-2025-10-29",
+    "user-profiles-2026-03-24": null,
+    "user-profiles-2026-08-18": null,
+    "user-profiles-2026-09-04": null,
+    "webhooks-2026-10-07": null,
+    "afk-mode-2026-01-31": "afk-mode-2026-01-31",
+    "redact-thinking-2026-02-12": null,
+    "structured-outputs-2025-12-15": "structured-outputs-2025-12-15"
   },
   "vertex_ai_header_only": {
+    "ce-plugins-2026-09-01": "ce-plugins-2026-09-01",
+    "ce-user-management-2026-07-13": "ce-user-management-2026-07-13",
     "compact-2026-09-04": "compact-2026-09-04",
-    "inline-tools-2026-09-15": "inline-tools-2026-09-15"
+    "dreaming-2026-04-21": "dreaming-2026-04-21",
+    "experimental-claude-tag-2026-10-21": "experimental-claude-tag-2026-10-21",
+    "extended-cache-ttl-2025-04-11": "extended-cache-ttl-2025-04-11",
+    "inline-tools-2026-09-15": "inline-tools-2026-09-15",
+    "managed-agents-2026-04-01": "managed-agents-2026-04-01",
+    "mcp-tunnels-2026-06-22": "mcp-tunnels-2026-06-22",
+    "oauth-2025-04-20": "oauth-2025-04-20",
+    "prompt-caching-2024-07-31": "prompt-caching-2024-07-31",
+    "prompt-caching-scope-2026-01-05": "prompt-caching-scope-2026-01-05",
+    "redact-thinking-2026-02-12": "redact-thinking-2026-02-12",
+    "spend-limit-reads-2026-09-26": "spend-limit-reads-2026-09-26",
+    "telemetry-destinations-2026-08-11": "telemetry-destinations-2026-08-11",
+    "user-profiles-2026-03-24": "user-profiles-2026-03-24",
+    "user-profiles-2026-08-18": "user-profiles-2026-08-18",
+    "user-profiles-2026-09-04": "user-profiles-2026-09-04"
   },
   "databricks": {
     "advisor-tool-2026-03-01": null,

litellm/batches/batch_utils.py

@@ -3,9 +3,10 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from dataclasses import replace as dataclasses_replace
 from enum import Enum
+from itertools import groupby
 from typing import Any, Final, Literal, cast
 
-from pydantic import TypeAdapter
+from pydantic import JsonValue, TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
@@ -15,6 +16,7 @@ from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
 from litellm.llms.bedrock.batches.transformation import titan_embedding_usage_from_batch_output
 from litellm.llms.vertex_ai.batches.transformation import (
     is_native_vertex_batch_output_row,
+    native_vertex_batch_row_model,
     native_vertex_batch_row_stats,
 )
 from litellm.types.llms.openai import Batch
@@ -210,6 +212,7 @@ class _BatchOutputLineStats:
     cache_creation_tokens: int
     reasoning_tokens: int
     model: str | None
+    usage: Usage | None
 
 
 def _classify_output_line_stats(
@@ -268,7 +271,7 @@ def _compute_output_line_stats(
     raw_model: Final = response_body.get("model")
     response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
     completion_details: Final = usage.completion_tokens_details
-    line_prompt_cost, line_completion_cost = _output_line_cost(
+    line_prompt_cost, line_completion_cost = _output_line_cost_or_pending(
         response_body=response_body,
         usage=usage,
         custom_llm_provider=custom_llm_provider,
@@ -286,7 +289,15 @@ def _compute_output_line_stats(
         cache_read_tokens=prompt_details["cache_hit_tokens"],
         cache_creation_tokens=prompt_details["cache_creation_tokens"],
         reasoning_tokens=(completion_details.reasoning_tokens if completion_details else None) or 0,
-        model=response_model,
+        model=response_model or model_name,
+        usage=(
+            usage.model_copy(update=ocr_usage.model_dump(exclude_none=True))
+            if (ocr_usage := _ocr_usage_info_from_response_body(response_body)) is not None
+            and ocr_usage.model_dump(exclude_none=True)
+            else usage
+            if response_body.get("usage") is not None or usage.total_tokens > 0
+            else None
+        ),
     )
 
 
@@ -298,6 +309,54 @@ def _ocr_usage_info_from_response_body(response_body: Mapping[str, object]) -> O
     return OCRUsageInfo.model_validate(raw_usage_info)
 
 
+def _output_line_cost_or_pending(
+    response_body: Mapping[str, object],
+    usage: Usage,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
+    model_name: str | None,
+    response_model: str | None,
+    model_info: ModelInfo | None,
+    vertex_location: str | None,
+) -> tuple[float, float]:
+    try:
+        return _output_line_cost(
+            response_body, usage, custom_llm_provider, model_name, response_model, model_info, vertex_location
+        )
+    except Exception:  # noqa: BLE001  # retain measured usage when existing batch pricing fails
+        verbose_logger.warning("Batch line pricing unavailable; retaining usage with the existing zero-cost fallback")
+        return 0.0, 0.0
+
+
+def _combined_reported_batch_usage(usages: tuple[Usage | None, ...]) -> Mapping[str, object] | None:
+    from litellm.cost_calculator import BaseTokenUsageProcessor
+
+    reported: Final = [usage for usage in usages if usage is not None]
+    if not reported:
+        return None
+    fields: Final = tuple(TypeAdapter(dict[str, JsonValue]).validate_python(usage.model_dump()) for usage in reported)
+    quantities: Final = {
+        key: sum(value for usage in fields if isinstance(value := usage.get(key), (int, float)))
+        for key in OCRUsageInfo.model_fields
+        if any(isinstance(usage.get(key), (int, float)) for usage in fields)
+    }
+    return {**BaseTokenUsageProcessor.combine_usage_objects(usage_objects=reported).model_dump(), **quantities}
+
+
+def _batch_usage_by_request(usages: tuple[tuple[str | None, Usage | None], ...]) -> tuple[Mapping[str, object], ...]:
+    rows: Final = tuple(
+        {"model": model, "usage": usage.model_dump() if usage is not None else None} for model, usage in usages
+    )
+    return tuple(sorted(rows, key=lambda row: json.dumps(row, sort_keys=True)))
+
+
+def _batch_usage_by_model(usages: tuple[tuple[str | None, Usage | None], ...]) -> Mapping[str, object]:
+    ordered: Final = sorted(usages, key=lambda item: item[0] or "")
+    return {
+        model: _combined_reported_batch_usage(tuple(item[1] for item in group))
+        for model, group in groupby(ordered, key=lambda item: item[0] or "")
+    }
+
+
 def _output_line_cost(
     response_body: Mapping[str, object],
     usage: Usage,
@@ -361,6 +420,9 @@ def _aggregate_batch_cost_usage_models(
         completion_tokens=sum(stats.completion_tokens for stats in line_stats),
         reasoning_tokens=sum(stats.reasoning_tokens for stats in line_stats),
         **cache_token_params,
+        batch_usage_by_model=_batch_usage_by_model(tuple((stats.model, stats.usage) for stats in line_stats)),
+        batch_usage_by_request=_batch_usage_by_request(tuple((stats.model, stats.usage) for stats in line_stats)),
+        batch_unknown_usage_requests=sum(stats.usage is None for stats in line_stats),
     )
     batch_models: Final = [model_name] if model_name else [stats.model for stats in line_stats if stats.model]
     total_prompt_cost: Final = sum((stats.prompt_cost for stats in line_stats), 0.0)
@@ -401,17 +463,21 @@ def calculate_vertex_ai_batch_cost_and_usage(
     from litellm.cost_calculator import batch_cost_calculator
     from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
 
-    row_stats: Final = tuple(
-        native_vertex_batch_row_stats(
-            row,
-            model_name,
-            model_info=model_info,
-            calculate_usage=VertexGeminiConfig.calculate_usage,
-            cost_calculator=batch_cost_calculator,
-            vertex_location=vertex_location,
+    observations: Final = tuple(
+        (
+            native_vertex_batch_row_model(row, model_name),
+            native_vertex_batch_row_stats(
+                row,
+                model_name,
+                model_info=model_info,
+                calculate_usage=VertexGeminiConfig.calculate_usage,
+                cost_calculator=batch_cost_calculator,
+                vertex_location=vertex_location,
+            ),
         )
         for row in vertex_ai_batch_responses
     )
+    row_stats: Final = tuple(stats for _, stats in observations)
     priced: Final = tuple(stats for stats in row_stats if stats is not None)
     total_prompt_cost: Final = sum(stats.prompt_cost for stats in priced)
     total_completion_cost: Final = sum(stats.completion_cost for stats in priced)
@@ -435,6 +501,13 @@ def calculate_vertex_ai_batch_cost_and_usage(
             total_tokens=total_tokens,
             prompt_tokens=prompt_tokens,
             completion_tokens=completion_tokens,
+            batch_usage_by_model=_batch_usage_by_model(
+                tuple((model, stats.usage if stats is not None else None) for model, stats in observations)
+            ),
+            batch_usage_by_request=_batch_usage_by_request(
+                tuple((model, stats.usage if stats is not None else None) for model, stats in observations)
+            ),
+            batch_unknown_usage_requests=sum(stats is None for stats in row_stats),
         ),
         models=(
             [model_name]

litellm/caching/caching.py

@@ -25,7 +25,11 @@ from litellm._internal_context import current_service_target, service_target
 from litellm._logging import verbose_logger
 from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
 from litellm.integrations.otel.runtime import phase_span
-from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases
+from litellm.litellm_core_utils.model_param_helper import (
+    ModelParamHelper,
+    embeds_thought_signature_in_tool_call_id,
+    peek_reasoning_summary_aliases,
+)
 from litellm.types.caching import *
 from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
 
@@ -421,7 +425,12 @@ class Cache:
         reasoning_summary_material: Final = (
             "" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
         )
-        hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
+        tool_call_id_shape_material: Final = (
+            "" if embeds_thought_signature_in_tool_call_id(kwargs) else "thought_signature_in_tool_call_id: False"
+        )
+        hashed_cache_key = Cache._get_hashed_cache_key(
+            cache_key + reasoning_summary_material + tool_call_id_shape_material
+        )
         hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
         verbose_logger.debug(
             "\nCreated cache key: %s (source material length: %d)",

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -34,10 +34,12 @@ from litellm.completion_extras.litellm_responses_transformation.history import (
     requested_phase,
 )
 from litellm.integrations.anthropic_cache_control_hook import supports_openai_prompt_cache_breakpoint
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.hidden_params import get_hidden_params, get_or_create_hidden_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
-    prompt_cache_breakpoint_for_wire,
+    prompt_cache_breakpoint_with_readable_mode,
     responses_reasoning_items_from_thinking_blocks,
+    whole_prompt_cache_breakpoint_or_none,
     with_prompt_cache_breakpoint,
 )
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
@@ -97,6 +99,12 @@ _RESPONSES_API_ONLY_FIELDS: Final = frozenset((*Response.model_fields, *Response
 )
 
 
+def _bridged_prompt_cache_breakpoint(marker: object, drop_params: bool, target_trims_breakpoints: bool) -> object:
+    if target_trims_breakpoints:
+        return prompt_cache_breakpoint_with_readable_mode(marker, drop_params)
+    return whole_prompt_cache_breakpoint_or_none(marker) if drop_params else marker
+
+
 def _strip_prompt_cache_breakpoint_from_content_block(value: object) -> object:
     if not isinstance(value, dict):
         return value
@@ -844,6 +852,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         messages: list["AllMessageValues"],
         *,
         drop_params: bool = False,
+        target_trims_breakpoints: bool = True,
         keep_prompt_cache_breakpoints: bool = False,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
@@ -852,6 +861,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         converted_input_items, instructions = self._convert_chat_completion_messages_to_responses_input(
             messages,
             drop_params=drop_params,
+            target_trims_breakpoints=target_trims_breakpoints,
             replay_tool_search=replay_tool_search,
             include_namespace=include_namespace,
             replay_openai_history=replay_openai_history,
@@ -868,6 +878,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         messages: list["AllMessageValues"],
         *,
         drop_params: bool = False,
+        target_trims_breakpoints: bool = True,
         replay_tool_search: bool = False,
         include_namespace: bool = False,
         replay_openai_history: bool = False,
@@ -921,6 +932,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                 content,
                                 role,
                                 drop_params=drop_params,
+                                target_trims_breakpoints=target_trims_breakpoints,
                             ),
                         }
                     )
@@ -940,6 +952,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         content,
                         "user",  # Use "user" role to get input_* types
                         drop_params=drop_params,
+                        target_trims_breakpoints=target_trims_breakpoints,
                     )
                 else:
                     # Fallback: convert unexpected types to input_text
@@ -968,7 +981,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             "type": "message",
                             "role": "assistant",
                             "content": self._convert_content_to_responses_format(
-                                content, "assistant", drop_params=drop_params
+                                content,
+                                "assistant",
+                                drop_params=drop_params,
+                                target_trims_breakpoints=target_trims_breakpoints,
                             ),
                             **_assistant_phase_field(msg, replay_openai_history, drop_params),
                         }
@@ -1009,7 +1025,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         "type": "message",
                         "role": role,
                         "content": self._convert_content_to_responses_format(
-                            content, cast(str, role), drop_params=drop_params
+                            content,
+                            cast(str, role),
+                            drop_params=drop_params,
+                            target_trims_breakpoints=target_trims_breakpoints,
                         ),
                         **_assistant_phase_field(msg, replay_openai_history and role == "assistant", drop_params),
                     }
@@ -1135,6 +1154,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         converted_input_items, converted_instructions = self.convert_chat_completion_messages_to_responses_api(
             messages,
             drop_params=bool(litellm_params.get("drop_params") or litellm.drop_params),
+            target_trims_breakpoints=drops_unsupported_params(litellm_params.get("drop_params")),
             keep_prompt_cache_breakpoints=supports_prompt_cache_breakpoint,
             replay_tool_search=_requests_tool_search(cast(object, optional_params)),  # cast-ok: upcast
             include_namespace=True,
@@ -1640,6 +1660,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         | None,
         role: str,
         drop_params: bool = False,
+        target_trims_breakpoints: bool = True,
     ) -> list[dict[str, object]]:
         """Convert chat completion content to responses API format"""
         from litellm.types.llms.openai import ChatCompletionImageObject
@@ -1666,7 +1687,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                     if original_type == "text":
                         converted = with_prompt_cache_breakpoint(
                             self._convert_content_str_to_input_text(item.get("text", ""), role),
-                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            _bridged_prompt_cache_breakpoint(
+                                item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                            ),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   text -> %s", converted)
@@ -1679,14 +1702,18 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                     cast(ChatCompletionImageObject, item), role
                                 ),
                             ),
-                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            _bridged_prompt_cache_breakpoint(
+                                item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                            ),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   image_url -> %s", converted)
                     elif original_type == "input_audio":
                         converted = with_prompt_cache_breakpoint(
                             {"type": "input_audio", "input_audio": item.get("input_audio")},
-                            prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                            _bridged_prompt_cache_breakpoint(
+                                item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                            ),
                         )
                         result.append(converted)
                         verbose_logger.debug("Chat provider:   input_audio -> %s", converted)
@@ -1702,7 +1729,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                                 _input_file_from_file_value(
                                     cast("ChatCompletionFileObject", item).get("file"),  # cast-ok: type tag checked
                                 ),
-                                prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                                _bridged_prompt_cache_breakpoint(
+                                    item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                                ),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   file -> %s", converted)
@@ -1726,7 +1755,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                             # Default to input_text for unknown types
                             converted = with_prompt_cache_breakpoint(
                                 self._convert_content_str_to_input_text(str(item.get("text", item)), role),
-                                prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params),
+                                _bridged_prompt_cache_breakpoint(
+                                    item.get("prompt_cache_breakpoint"), drop_params, target_trims_breakpoints
+                                ),
                             )
                             result.append(converted)
                             verbose_logger.debug("Chat provider:   unknown(%s) -> %s", original_type, converted)

litellm/constants.py

@@ -432,6 +432,7 @@ WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
 DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
 NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
 DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
+DEEPGRAM_SPEAK_DEFAULT_MODEL: Final = "aura-asteria-en"
 
 BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
 BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
@@ -675,7 +676,7 @@ request_timeout: float = (
     DEFAULT_REQUEST_TIMEOUT_SECONDS if _request_timeout_from_env is None else _request_timeout_from_env
 )
 request_timeout_explicitly_set: bool = _request_timeout_from_env is not None
-DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0)  # 10 minutes
+DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0)
 AGENT_KILL_SWITCH_TIMEOUT_SECONDS: Final = 10.0
 AGENT_KILL_SWITCH_RESPONSE_BODY_MAX_CHARS: Final = 2000
 # Patterns that indicate a localhost/internal URL in A2A agent cards that should be
@@ -1691,6 +1692,7 @@ BAGGAGE_MAPPED_TAGS_METADATA_KEY: Final = "_baggage_mapped_tags"
 VECTOR_STORE_CALLER_METADATA_KWARG: Final = "litellm_vector_store_caller_metadata"
 ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY: Final = "_litellm_router_usage_counted_tokens"
 INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
+RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY: Final = "responses_websocket_call_id"
 BILLED_RETRY_ATTEMPT_METADATA_KEY: Final = "billed_retry_attempt"
 MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
 SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"

litellm/cost_calculator.py

@@ -6,6 +6,7 @@ import time
 from collections.abc import Mapping, Sequence
 from datetime import datetime
 from functools import lru_cache
+from itertools import accumulate
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, TypeAlias, cast
 
@@ -1554,12 +1555,66 @@ def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float
     return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
 
 
+def _cost_map_priced_input_output(
+    model: str | None,
+    custom_llm_provider: str | None,
+    usage: Usage | None,
+    call_type: CallTypesLiteral | None,
+    service_tier: str | None,
+    data_residency: str | None,
+) -> tuple[float, float]:
+    if not model or usage is None:
+        return 0.0, 0.0
+    try:
+        return cost_per_token(
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            usage_object=usage,
+            call_type=call_type or "completion",
+            service_tier=service_tier,
+            data_residency=data_residency,
+        )
+    except Exception as pricing_error:  # noqa: BLE001  # cost_per_token raises plain Exception for an unmapped model
+        verbose_logger.debug("No cost map price for reported-cost split of %s: %s", model, pricing_error)
+        return 0.0, 0.0
+
+
+def split_reported_cost_by_cost_map_ratio(
+    reported_total: float,
+    model: str | None,
+    custom_llm_provider: str | None,
+    usage: Usage | None,
+    call_type: CallTypesLiteral | None,
+    service_tier: str | None,
+    data_residency: str | None,
+) -> tuple[float, float]:
+    """Apportions a provider-reported total between input and output in the ratio litellm's own cost map would
+    price the same usage, so a per-input-token model books the whole total on input. Unmapped or zero-priced
+    usage keeps the whole total on output."""
+    priced_input, priced_output = _cost_map_priced_input_output(
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        usage=usage,
+        call_type=call_type,
+        service_tier=service_tier,
+        data_residency=data_residency,
+    )
+    priced_total: Final = priced_input + priced_output
+    if priced_total <= 0:
+        return 0.0, reported_total
+    input_cost: Final = reported_total * (priced_input / priced_total)
+    return input_cost, reported_total - input_cost
+
+
 def _adjusted_provider_billed_cost(
     reported_cost: float,
     custom_llm_provider: str | None,
+    pricing_provider: str | None,
     model: str | None,
     region_name: str | None,
     litellm_logging_obj: LitellmLoggingObject | None,
+    usage: Usage | None,
+    call_type: CallTypesLiteral | None,
     service_tier: str | None,
     data_residency: str | None,
 ) -> float:
@@ -1569,10 +1624,19 @@ def _adjusted_provider_billed_cost(
     adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin(
         base_cost=discounted_cost, custom_llm_provider=custom_llm_provider
     )
+    input_cost, output_cost = split_reported_cost_by_cost_map_ratio(
+        reported_total=adjusted_cost,
+        model=model,
+        custom_llm_provider=pricing_provider,
+        usage=usage,
+        call_type=call_type,
+        service_tier=service_tier,
+        data_residency=data_residency,
+    )
     _store_cost_breakdown_in_logging_obj(
         litellm_logging_obj=litellm_logging_obj,
-        prompt_tokens_cost_usd_dollar=0.0,
-        completion_tokens_cost_usd_dollar=adjusted_cost,
+        prompt_tokens_cost_usd_dollar=input_cost,
+        completion_tokens_cost_usd_dollar=output_cost,
         cost_for_built_in_tools_cost_usd_dollar=0.0,
         total_cost_usd_dollar=adjusted_cost,
         original_cost=reported_cost,
@@ -2080,9 +2144,16 @@ def completion_cost(
                     return _adjusted_provider_billed_cost(
                         reported_cost=reported_cost,
                         custom_llm_provider=custom_llm_provider,
+                        pricing_provider=_pricing_provider_for_cost_candidate(
+                            candidate_provider=candidate_pricing_provider,
+                            request_provider=request_pricing_provider,
+                            current_provider=custom_llm_provider,
+                        ),
                         model=model_for_discount,
                         region_name=region_name if isinstance(region_name, str) else None,
                         litellm_logging_obj=litellm_logging_obj,
+                        usage=cost_per_token_usage_object,
+                        call_type=call_type,
                         service_tier=service_tier,
                         data_residency=data_residency,
                     )
@@ -2387,10 +2458,28 @@ def response_cost_calculator(
                     provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
                     if provider_response_cost is not None and not custom_pricing:
                         if _breakdown_total(litellm_logging_obj) != provider_response_cost:
+                            header_priced_model: Final = select_model_name_for_cost_calc(
+                                model=model,
+                                completion_response=response_object,
+                                custom_llm_provider=custom_llm_provider,
+                                custom_pricing=custom_pricing,
+                                base_model=base_model,
+                                router_model_id=router_model_id,
+                                region_name=region_name,
+                            )
+                            header_input_cost, header_output_cost = split_reported_cost_by_cost_map_ratio(
+                                reported_total=provider_response_cost,
+                                model=header_priced_model or model,
+                                custom_llm_provider=pricing_provider or custom_llm_provider,
+                                usage=get_usage_object(response_object),
+                                call_type=call_type,
+                                service_tier=service_tier,
+                                data_residency=data_residency,
+                            )
                             _store_cost_breakdown_in_logging_obj(
                                 litellm_logging_obj=litellm_logging_obj,
-                                prompt_tokens_cost_usd_dollar=0.0,
-                                completion_tokens_cost_usd_dollar=provider_response_cost,
+                                prompt_tokens_cost_usd_dollar=header_input_cost,
+                                completion_tokens_cost_usd_dollar=header_output_cost,
                                 cost_for_built_in_tools_cost_usd_dollar=0.0,
                                 total_cost_usd_dollar=provider_response_cost,
                                 original_cost=provider_response_cost,
@@ -3319,7 +3408,7 @@ class RealtimeAPITokenUsageProcessor(BaseTokenUsageProcessor):
         )
 
 
-_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete"})
+_RESPONSES_WS_BILLABLE_EVENT_TYPES: Final = frozenset({"response.completed", "response.incomplete", "response.failed"})
 
 
 class _ResponsesWsEventResponse(LiteLLMBaseModel):
@@ -3762,44 +3851,65 @@ def handle_realtime_transcription_cost_calculation(
       - {"type": "duration", "seconds": <float>}  → priced via input_cost_per_second
       - {"type": "tokens", "input_tokens": ...}    → priced via input/audio token cost
     """
-    completed_events: Final = [
-        cast(dict, result) for result in results if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
-    ]
-    if not completed_events:
+    configured_models: Final = tuple(accumulate(map(_session_transcription_model, results), _newer_model))
+    clips: Final = tuple(
+        (as_str_mapping(result.get("usage")) or {}, configured)
+        for result, configured in zip(results, configured_models, strict=True)
+        if result.get("type") == _TRANSCRIPTION_COMPLETED_EVENT_TYPE
+    )
+    if not clips:
         return 0.0
 
-    model_name: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
-    model_info: Final = _cost_map_model_info(model_name, custom_llm_provider)
+    fallback_model: Final = _get_transcription_model_name_from_results(results) or litellm_model_name
+    clip_models: Final = tuple((usage, configured or fallback_model) for usage, configured in clips)
+    model_infos: Final = MappingProxyType(
+        {model: _cost_map_model_info(model, custom_llm_provider) for model in frozenset(m for _, m in clip_models)}
+    )
     override_info: Final = (
         _cost_map_model_info(custom_pricing_model, custom_llm_provider) if custom_pricing_model is not None else None
     )
+    return sum(
+        (_transcription_usage_cost(dict(usage), model_infos[model], override_info) for usage, model in clip_models),
+        0.0,
+    )
 
-    total_cost = 0.0
-    for event in completed_events:
-        usage = event.get("usage") or {}
-        total_cost += _transcription_usage_cost(usage, model_info, override_info)
-    return total_cost
+
+_TRANSCRIPTION_SESSION_EVENT_TYPES: Final = frozenset(
+    {"transcription_session.created", "transcription_session.updated", "session.created", "session.updated"}
+)
+
+
+def _str_field(mapping: Mapping[str, object] | None, key: str) -> str | None:
+    value: Final = None if mapping is None else mapping.get(key)
+    return value if isinstance(value, str) and value else None
+
+
+def _configured_transcription_model(session: Mapping[str, object]) -> str | None:
+    audio_input: Final = as_str_mapping((as_str_mapping(session.get("audio")) or {}).get("input"))
+    nested: Final = _str_field(as_str_mapping((audio_input or {}).get("transcription")), "model")
+    return nested or _str_field(as_str_mapping(session.get("input_audio_transcription")), "model")
+
+
+def _session_of(result: Mapping[str, object]) -> Mapping[str, object] | None:
+    return as_str_mapping(result.get("session")) if result.get("type") in _TRANSCRIPTION_SESSION_EVENT_TYPES else None
+
+
+def _session_transcription_model(result: Mapping[str, object]) -> str | None:
+    session: Final = _session_of(result)
+    return None if session is None else _configured_transcription_model(session)
+
+
+def _newer_model(current: str | None, newer: str | None) -> str | None:
+    return newer or current
 
 
 def _get_transcription_model_name_from_results(
     results: OpenAIRealtimeStreamList,
 ) -> str | None:
-    """Resolve the ASR model from a transcription_session.* / session.* event."""
-    for result in results:
-        if result.get("type") in (
-            "transcription_session.created",
-            "transcription_session.updated",
-            "session.created",
-            "session.updated",
-        ):
-            session = cast(dict, result).get("session", {}) or {}
-            transcription = ((session.get("audio", {}) or {}).get("input", {}) or {}).get(
-                "transcription", {}
-            ) or session.get("input_audio_transcription", {})
-            model = (transcription or {}).get("model") or session.get("model")
-            if model:
-                return model
-    return None
+    return next(
+        (model for result in results if (model := _str_field(_session_of(result), "model"))),
+        None,
+    )
 
 
 _TranscriptionRateKey: TypeAlias = Literal[

litellm/endpoints/speech/speech_to_completion_bridge/handler.py

@@ -113,7 +113,7 @@ class SpeechToCompletionBridgeHandler:
             voice=voice,
         )
 
-        with internal_sub_call():
+        with internal_sub_call(logging_obj.litellm_call_id):
             result: Final = completion(
                 **request_data,
             )

litellm/guardrails.py

@@ -0,0 +1,66 @@
+"""Invoke configured gateway guardrails without generating a completion."""
+
+from collections.abc import Mapping, Sequence
+from enum import Enum
+from typing import Final
+
+import httpx
+from pydantic import TypeAdapter
+
+import litellm
+from litellm.llms.custom_httpx.http_handler import (
+    get_async_httpx_client,  # pyright: ignore[reportUnknownVariableType]  # shared factory has untyped optional parameters
+)
+from litellm.types.guardrails import NativeGuardrailResponse
+
+_NATIVE_RESPONSE: Final[TypeAdapter[NativeGuardrailResponse]] = TypeAdapter(NativeGuardrailResponse)
+
+
+class _InputType(Enum):
+    REQUEST = "request"
+    RESPONSE = "response"
+
+
+async def apply(
+    guardrail_name: str,
+    *,
+    input: str | None = None,
+    response: str | None = None,
+    messages: Sequence[Mapping[str, object]] | None = None,
+    metadata: Mapping[str, object] | None = None,
+    api_base: str | None = None,
+    api_key: str | None = None,
+    timeout: float | httpx.Timeout | None = None,
+) -> NativeGuardrailResponse:
+    """Return a configured guardrail's native structured findings.
+
+    Supply exactly one of ``input`` or ``response``. Grounding guardrails can
+    use ``messages`` as evidence. Connection settings default to
+    ``litellm.api_base`` and ``litellm.api_key``.
+    """
+    if (input is None) == (response is None):
+        raise ValueError("Supply exactly one of input or response.")
+    resolved_base: Final = api_base if api_base is not None else litellm.api_base
+    if not resolved_base:
+        raise ValueError("Set api_base or litellm.api_base to your LiteLLM gateway URL.")
+    resolved_key: Final = api_key if api_key is not None else litellm.api_key
+    input_type: Final = _InputType.RESPONSE if response is not None else _InputType.REQUEST
+    payload: Final = dict[str, object](
+        guardrail_name=guardrail_name,
+        text=response if response is not None else input,
+        input_type=input_type.value,
+        response_format="native",
+    )
+    if messages is not None:
+        payload["messages"] = [dict(message) for message in messages]
+    if metadata is not None:
+        payload["metadata"] = dict(metadata)
+    gateway_root: Final = resolved_base.rstrip("/").removesuffix("/v1")
+    client: Final = get_async_httpx_client(llm_provider="litellm_guardrails")
+    result: Final = await client.post(  # pyright: ignore[reportUnknownMemberType]  # shared handler has untyped optional parameters
+        url=f"{gateway_root}/guardrails/apply_guardrail",
+        json=payload,
+        headers={"Authorization": f"Bearer {resolved_key}"} if resolved_key else {},
+        timeout=timeout,
+    )
+    return _NATIVE_RESPONSE.validate_json(result.content)

litellm/integrations/anthropic_cache_control_hook.py

@@ -13,11 +13,11 @@ import copy
 import itertools
 import os
 import re
-from collections.abc import Callable, Container, Iterable, Mapping, Sequence
+from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
 from urllib.parse import urlparse
 
-from pydantic import TypeAdapter, ValidationError
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
 from litellm.integrations.custom_logger import CustomLogger
@@ -32,6 +32,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
     tool_call_ids_answered_by_client,
     tool_call_ids_answered_in,
 )
+from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import string_field
 from litellm.llms.anthropic.common_utils import (
     is_claude_code_one_shot_subagent_request,
     supports_anthropic_cache_control,
@@ -72,7 +73,7 @@ MAX_CACHE_CONTROL_BLOCKS: Final = 4
 
 CACHE_BREAKPOINT_KEYS: Final = ("cache_control", "prompt_cache_breakpoint")
 OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION: Final = (5, 6)
-_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+))?")
+_GPT_VERSION_PATTERN: Final = re.compile(r"^gpt-(\d+)(?:\.(\d+)|-(0|[1-9]\d?)(?![\da-z]))?")
 _FINE_TUNED_BASE_PATTERN: Final = re.compile(r"^ft:([^:]+)")
 _O_SERIES_PATTERN: Final = re.compile(r"^o\d+(?:-|$)")
 OPENAI_PROMPT_CACHE_BREAKPOINT_BLOCK_TYPES: Final = frozenset(
@@ -128,7 +129,7 @@ def _message_mark_reaches_provider(fields: Mapping[str, object]) -> bool:
         return False
     return (
         not isinstance(fields.get("content"), list)
-        or fields.get("role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
+        or string_field(fields, "role") in _ROLES_SENDING_MESSAGE_MARK_WITH_LIST_CONTENT
     )
 
 
@@ -334,6 +335,30 @@ def openai_prompt_cache_breakpoint_support(model: str, custom_llm_provider: str
     return _named_model_breakpoint_support(model)
 
 
+class _DeclaredBaseModels(BaseModel):
+    model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
+
+    base_model: object = None
+    model_info: object = None
+
+    def names(self) -> tuple[str, ...]:
+        model_info: Final = _validated_object_mapping(self.model_info) or {}
+        candidates: Final = (self.base_model, model_info.get("base_model"))
+        return tuple(name for name in candidates if isinstance(name, str) and name)
+
+
+def deployment_prompt_cache_breakpoint_support(
+    model: str, custom_llm_provider: str, litellm_params: object
+) -> bool | None:
+    """``openai_prompt_cache_breakpoint_support`` for a deployment, read from its declared base model first."""
+    declared: Final = _DeclaredBaseModels.model_validate(litellm_params)
+    supports: Final = (
+        openai_prompt_cache_breakpoint_support(name, custom_llm_provider)
+        for name in dict.fromkeys((*declared.names(), model))
+    )
+    return next((support for support in supports if support is not None), None)
+
+
 def _named_model_breakpoint_support(model: str) -> bool | None:
     bare_model: Final = model.rsplit("/", 1)[-1].lower()
     if _O_SERIES_PATTERN.match(bare_model) is not None:
@@ -341,7 +366,7 @@ def _named_model_breakpoint_support(model: str) -> bool | None:
     version_match: Final = _GPT_VERSION_PATTERN.match(bare_model)
     if version_match is None:
         return None
-    version: Final = (int(version_match.group(1)), int(version_match.group(2) or 0))
+    version: Final = (int(version_match.group(1)), int(version_match.group(2) or version_match.group(3) or 0))
     return version >= OPENAI_PROMPT_CACHE_BREAKPOINT_MIN_GPT_VERSION
 
 
@@ -445,22 +470,41 @@ def _as_object_list(value: object | None) -> list[object] | None:
     return _validated_object_list(value)
 
 
-def _tool_call_carries_cache_breakpoint(tool_call: object, message: object, answered_tool_ids: Container[str]) -> bool:
+def _server_tool_call_id(tool_call: object) -> str | None:
+    tool_call_id: Final = _attribute_or_key(tool_call, "id")
+    if isinstance(tool_call_id, str) and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX):
+        return tool_call_id
+    return None
+
+
+def _tool_call_carries_cache_breakpoint(tool_call: object, client_tool_call_ids: Container[str]) -> bool:
     if _attribute_or_key(tool_call, "cache_control") is None:
         return False
+    server_tool_call_id: Final = _server_tool_call_id(tool_call)
+    return server_tool_call_id is None or server_tool_call_id in client_tool_call_ids
 
-    return not tool_call_is_rebuilt_as_server_tool_use(
-        _attribute_or_key(tool_call, "id"), _attribute_or_key(message, "provider_specific_fields"), answered_tool_ids
+
+def _server_tool_call_ids_without_a_kept_result(message: object) -> Iterator[str]:
+    provider_specific_fields: Final = _attribute_or_key(message, "provider_specific_fields")
+    tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or ()
+    return (
+        tool_call_id
+        for tool_call in tool_calls
+        if (tool_call_id := _server_tool_call_id(tool_call)) is not None
+        and not tool_call_is_rebuilt_as_server_tool_use(tool_call_id, provider_specific_fields)
     )
 
 
-def _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(
+def _client_tool_call_ids_when_a_server_tool_call_is_marked(
     messages: Sequence[object], model: str | None = None, custom_llm_provider: str | None = None
 ) -> frozenset[str]:
     if not _marks_a_server_tool_call(messages):
         return frozenset()
     if model is not None and not _builds_anthropic_messages(model, custom_llm_provider):
-        return tool_call_ids_answered_in(messages)
+        server_calls_sent_as_client_calls: Final = itertools.chain.from_iterable(
+            map(_server_tool_call_ids_without_a_kept_result, messages)
+        )
+        return tool_call_ids_answered_in(messages) | frozenset(server_calls_sent_as_client_calls)
     return tool_call_ids_answered_by_client(messages, _keeps_mid_conversation_system(model, custom_llm_provider))
 
 
@@ -517,12 +561,7 @@ def _model_info_flags_mid_conversation_system(model: str, custom_llm_provider: s
 
 
 def _is_marked_server_tool_call(tool_call: object) -> bool:
-    tool_call_id: Final = _attribute_or_key(tool_call, "id")
-    return (
-        isinstance(tool_call_id, str)
-        and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
-        and _attribute_or_key(tool_call, "cache_control") is not None
-    )
+    return _server_tool_call_id(tool_call) is not None and _attribute_or_key(tool_call, "cache_control") is not None
 
 
 def _tool_carries_cache_breakpoint(tool: object) -> bool:
@@ -668,7 +707,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         reserved_blocks: Final = AnthropicCacheControlHook._blocks_reserved_outside_messages(
             remaining_points, external_breakpoints, openai_dialect
         )
-        answered_tool_ids: Final = _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(
+        answered_tool_ids: Final = _client_tool_call_ids_when_a_server_tool_call_is_marked(
             processed_messages, model, custom_llm_provider
         )
         breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(
@@ -757,7 +796,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         message_list: Final = tuple(messages)
         client_answers: Final = (
-            _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(message_list)
+            _client_tool_call_ids_when_a_server_tool_call_is_marked(message_list)
             if answered_tool_ids is None
             else answered_tool_ids
         )
@@ -849,7 +888,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         so earlier points win when slots are scarce.
         """
         client_answers: Final = (
-            _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(messages)
+            _client_tool_call_ids_when_a_server_tool_call_is_marked(messages)
             if answered_tool_ids is None
             else answered_tool_ids
         )
@@ -939,11 +978,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         content_count: Final = sum(1 for block in content if _carries_cache_breakpoint(block)) if content else 0
         tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls"))
         tool_call_count: Final = (
-            sum(
-                1
-                for tool_call in tool_calls
-                if _tool_call_carries_cache_breakpoint(tool_call, message, answered_tool_ids)
-            )
+            sum(1 for tool_call in tool_calls if _tool_call_carries_cache_breakpoint(tool_call, answered_tool_ids))
             if tool_calls
             else 0
         )

litellm/litellm_core_utils/core_helpers.py

@@ -4,7 +4,7 @@ import copy
 import logging
 import math
 import re
-from collections.abc import Collection, Iterable, Iterator, Mapping
+from collections.abc import Collection, Iterable, Iterator, Mapping, Sequence
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
 
@@ -104,6 +104,10 @@ def as_str_mapping(value: object) -> Mapping[str, object] | None:
     return value if isinstance(value, Mapping) else None
 
 
+def as_object_list(value: object) -> Sequence[object] | None:
+    return value if isinstance(value, list) else None
+
+
 def decoded_json_object(value: object) -> Mapping[str, object]:
     mapping: Final = as_str_mapping(value)
     if mapping is not None:
@@ -311,9 +315,12 @@ _FINISH_REASON_MAP: Final[dict[str, OpenAIChatCompletionFinishReason]] = {
     "tool_use": "tool_calls",
     "refusal": "content_filter",
     "compaction": "length",
+    "pause_turn": "length",
     "model_context_window_exceeded": "length",
     # Cohere
     "COMPLETE": "stop",
+    "STOP_SEQUENCE": "stop",
+    "TOOL_CALL": "tool_calls",
     "ERROR_TOXIC": "content_filter",
     "ERROR": "stop",
     # HuggingFace / Together AI
@@ -1073,9 +1080,14 @@ def billed_usage_cost(response: object, custom_llm_provider: str | None) -> floa
 _MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"})
 
 
+def reports_native_messages_cost(custom_llm_provider: object) -> bool:
+    """Whether a Messages upstream is one whose own reported cost litellm bills from instead of the price map."""
+    return isinstance(custom_llm_provider, str) and custom_llm_provider in _MESSAGES_REPORTED_COST_PROVIDERS
+
+
 def native_messages_reported_cost(usage: object, custom_llm_provider: object) -> float | None:
     """Only a Messages upstream known to price its own responses is trusted, every other one keeps the price map."""
-    if not isinstance(custom_llm_provider, str) or custom_llm_provider not in _MESSAGES_REPORTED_COST_PROVIDERS:
+    if not isinstance(custom_llm_provider, str) or not reports_native_messages_cost(custom_llm_provider):
         return None
     return provider_reported_usage_cost(usage, custom_llm_provider)
 

litellm/litellm_core_utils/get_litellm_params.py

@@ -84,6 +84,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
             "github_copilot_host",
             "github_copilot_user_session",
             "fireworks_forward_user_id",
+            "thought_signature_in_tool_call_id",
             "responses_api_version",
             "allowed_openai_params",
             PROVIDER_AFFINITY_HEADER_KWARG_KEY,
@@ -99,6 +100,8 @@ OPTIONAL_KWARGS_KEYS: Final = (
 # Backward-compatible alias for existing imports/tests.
 _OPTIONAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS
 
+SUPPLEMENTAL_KWARGS_KEYS: Final = OPTIONAL_KWARGS_KEYS | {"provider_specific_header", "extra_headers"}
+
 _CONTROL_OPTIONS: Final = TypeAdapter(ControlOptions)
 _CONTROL_OPTION_NAMES: Final = tuple(field.name for field in fields(ControlOptions))
 _MAX_SHOWN_INT_BITS: Final = 64

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -146,6 +146,10 @@ def handle_anthropic_text_model_custom_llm_provider(
     return model, custom_llm_provider
 
 
+def declared_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None:
+    return custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None)
+
+
 def declared_authenticating_provider(model: str | None, custom_llm_provider: str | None = None) -> str | None:
     """The authenticating provider this pair already names, or None.
 
@@ -154,7 +158,7 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str
     and for a declared pair the resolver's answer is the declaration itself, so metadata callers
     adopt the declaration instead of resolving.
     """
-    declared: Final = custom_llm_provider or (model.split("/", 1)[0] if model and "/" in model else None)
+    declared: Final = declared_provider(model, custom_llm_provider)
     return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None
 
 

litellm/litellm_core_utils/litellm_logging.py

@@ -16,9 +16,10 @@ import traceback
 from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from datetime import datetime as dt_object
+from enum import Enum
 from functools import lru_cache
 from types import MappingProxyType, TracebackType
-from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast
 
 from httpx import Response, Timeout
 from pydantic import BaseModel, JsonValue, TypeAdapter
@@ -47,6 +48,7 @@ from litellm.constants import (
     KUBERNETES_POD_ROUTING_KEY,
     LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
     PROVIDER_REQUEST_ID_HEADERS,
+    REALTIME_METERED_USAGE_KEY,
     REDACTED_BY_LITELLM,
 )
 from litellm.cost_calculator import (
@@ -84,6 +86,7 @@ from litellm.litellm_core_utils.core_helpers import (
     proxy_stamped_kubernetes_pod_routing,
     proxy_stamped_used_client_oauth_token,
     reconstruct_model_name,
+    reports_native_messages_cost,
     set_response_cost_in_hidden_params,
     single_log_line,
 )
@@ -163,6 +166,7 @@ from litellm.types.llms.openai import (
     ResponseIncompleteEvent,
     ResponsesAPIResponse,
 )
+from litellm.types.llms.vertex_ai import GenerateContentResponseBody, UsageMetadata
 from litellm.types.mcp import MCPPostCallResponseObject
 from litellm.types.prompts.init_prompts import PromptSpec
 from litellm.types.rerank import RerankResponse, get_rerank_usage
@@ -266,6 +270,9 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.llm_cost_calc.utils import BilledTokenRates
     from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector
     from litellm.proxy.hooks.autorouter_baseline_cache import BaselineCacheContext, CapturedBaselineObservation
+    from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation
+    from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
+    from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
 try:
     from litellm_enterprise.enterprise_callbacks.callback_controls import (
         EnterpriseCallbackControls,
@@ -571,6 +578,7 @@ def _resolve_mantle_region_for_cost(
 _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
 _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
 _SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
+_GOOGLE_USAGE_FIELDS: Final = frozenset(UsageMetadata.__annotations__)
 _REDACTED_RESULT: Final = TypeAdapter(object)
 _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
 
@@ -630,7 +638,60 @@ def same_attempt_marker(first: object, second: object) -> bool:
     return first is second or first == second
 
 
+class UsageObservation(Enum):
+    KNOWN = "known"
+    UNKNOWN = "unknown"
+    ESTIMATED = "estimated"
+
+
+@dataclass(frozen=True, slots=True)
+class UsageCaptureObservation:
+    usage_json: str | None
+    provenance: UsageObservation
+    response_cost: float | None = None
+
+    @classmethod
+    def from_usage(
+        cls, usage: Usage | None, provenance: UsageObservation, response_cost: float | None = None
+    ) -> "UsageCaptureObservation | UsageCaptureError":
+        from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
+
+        try:
+            serialized: Final = (
+                None if usage is None or provenance is UsageObservation.UNKNOWN else usage.model_dump_json()
+            )
+        except Exception as error:  # noqa: BLE001  # observation failures must not interrupt legacy retry handling
+            return UsageCaptureError(str(error))
+        else:
+            return cls(serialized, provenance, response_cost)
+
+
 class Logging(LiteLLMLoggingBaseClass):
+    usage_capture_callback: ClassVar[
+        Callable[
+            ["Logging", object, datetime.datetime, datetime.datetime, bool], "CapturedUsage | UsageCaptureError | None"
+        ]
+        | None
+    ] = None
+
+    def capture_usage_before_logging(
+        self,
+        response: object,
+        start_time: datetime.datetime | None = None,
+        end_time: datetime.datetime | None = None,
+        *,
+        freeze_only: bool = False,
+    ) -> None:
+        capture: Final = type(self).usage_capture_callback
+        if capture is not None and (not self.stream or self._is_assembled_stream_success(response)):
+            capture(
+                self,
+                response,
+                start_time or self.start_time,
+                end_time or datetime.datetime.now(datetime.timezone.utc),
+                freeze_only,
+            )
+
     global \
         supabaseClient, \
         promptLayerLogger, \
@@ -828,6 +889,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self.finalized_cost: FinalizedCost | None = None
         self.prepared_cost: PreparedCost | None = None
         self.cost_allowance_discount: float | None = None
+        self.cost_allowance_allocation: SearchAllowanceAllocation | None = None
         self.cost_finalization_lock = asyncio.Lock()
         self.deferred_success_result: tuple[object, datetime.datetime, datetime.datetime] | None = None
         self.billed_token_rates: BilledTokenRates | None = None
@@ -884,7 +946,7 @@ class Logging(LiteLLMLoggingBaseClass):
         new_call.start_time = start_time
         new_call.litellm_params = litellm_params
         new_call.model_call_details = {
-            **self.model_call_details,
+            **{key: value for key, value in self.model_call_details.items() if not key.startswith("_accounting_")},
             "litellm_call_id": litellm_call_id,
             "start_time": start_time,
             "first_api_call_start_time": None,
@@ -894,6 +956,7 @@ class Logging(LiteLLMLoggingBaseClass):
         new_call.finalized_cost = None
         new_call.prepared_cost = None
         new_call.cost_allowance_discount = None
+        new_call.cost_allowance_allocation = None
         new_call.cost_finalization_lock = asyncio.Lock()
         new_call.deferred_success_result = None
         new_call.streaming_chunks = []
@@ -2424,7 +2487,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
     _is_sync_litellm_request = is_sync_litellm_request
 
-    def _is_assembled_stream_success(self, result=None) -> bool:
+    def _is_assembled_stream_success(self, result: object = None) -> bool:
         """Final assembled stream export (not a per-chunk success call).
 
         Per-chunk callers pass a ``ModelResponseStream`` (or ``None``); the
@@ -2459,7 +2522,7 @@ class Logging(LiteLLMLoggingBaseClass):
         """
         from litellm.litellm_core_utils.thread_pool_executor import executor
 
-        if self._is_assembled_stream_success(result):
+        if self._is_assembled_stream_success(TypeAdapter(object).validate_python(result)):
             if self.model_call_details.get("has_dispatched_final_stream_success"):
                 return
             self.model_call_details["has_dispatched_final_stream_success"] = True
@@ -2477,6 +2540,11 @@ class Logging(LiteLLMLoggingBaseClass):
             )
             return
 
+        self.capture_usage_before_logging(
+            TypeAdapter(object).validate_python(result),
+            TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(start_time),
+            TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(end_time),
+        )
         await self.async_success_handler(
             result,
             start_time=start_time,
@@ -2503,6 +2571,17 @@ class Logging(LiteLLMLoggingBaseClass):
             return self.prepared_cost
         recorded_cache_hit: Final[object] = self.model_call_details.get("cache_hit")
         deferred: Final = self.deferred_success_result
+        observed_response: Final = _LITELLM_PARAMS.validate_python(
+            _extract_response_obj_and_hidden_params(deferred[0] if deferred else response, None)[0]
+        )
+        self.model_call_details["_accounting_usage_observed"] = (
+            UsageObservation.KNOWN
+            if (
+                self.model_call_details.get("combined_usage_object")
+                or StandardLoggingPayloadSetup.get_usage_as_dict(observed_response)
+            )
+            else UsageObservation.UNKNOWN
+        )
         prepared: Final = self.prepare_success_logging(
             result=deferred[0] if deferred is not None else response,
             start_time=deferred[1] if deferred is not None else self.start_time,
@@ -2540,17 +2619,29 @@ class Logging(LiteLLMLoggingBaseClass):
             },
             usage,
         )
-        self.record_partial_usage_for_failure(usage, raw_cost)
+        self.record_partial_usage_for_failure(
+            usage, raw_cost, observation=self.model_call_details["_accounting_usage_observed"]
+        )
         return self.prepared_cost
 
     def record_finalized_cost(self, finalized: FinalizedCost) -> None:
         self.finalized_cost = finalized
         self.model_call_details["response_cost"] = finalized.amount
 
-    def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None:
+    def record_usage_capture(self, captured: "CapturedUsage") -> None:
+        self.model_call_details["_accounting_capture"] = captured
+
+    def record_usage_pricing(self, pricing: object) -> None:
+        self.model_call_details["_accounting_pricing_snapshot"] = pricing
+
+    def record_partial_usage_for_failure(
+        self, usage: Usage, response_cost: float, *, observation: UsageObservation = UsageObservation.KNOWN
+    ) -> None:
         """Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
         self.model_call_details["combined_usage_object"] = usage
         self.model_call_details["response_cost"] = response_cost
+        if type(self).usage_capture_callback is not None:
+            self.model_call_details["_accounting_usage_observed"] = observation
 
     def record_failure_after_earlier_rounds(
         self, earlier: BilledRoundsSpend, failed_usage: Usage, failed_cost: float
@@ -2630,6 +2721,11 @@ class Logging(LiteLLMLoggingBaseClass):
         own usage while the request's failure log still bills every attempt."""
         usage: Final = self._model_call_details_view.get("combined_usage_object")
         response_cost: Final = self._model_call_details_view.get("response_cost")
+        if isinstance(usage, Usage) and type(self).usage_capture_callback is not None:
+            self.capture_failed_attempt_usage(usage)
+        self.model_call_details.pop("_accounting_capture", None)
+        self.model_call_details.pop("_accounting_pricing_snapshot", None)
+        self.model_call_details.pop("_accounting_usage_observed", None)
         self.model_call_details.pop("combined_usage_object", None)
         self.model_call_details.pop("response_cost", None)
         if isinstance(usage, Usage):
@@ -2639,6 +2735,53 @@ class Logging(LiteLLMLoggingBaseClass):
                 (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
             )
 
+    def capture_failed_attempt_usage(self, usage: Usage) -> None:
+        from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY
+        from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
+
+        capture: Final = type(self).usage_capture_callback
+        if capture is None:
+            return
+        attempt: Final = self.attempt_scoped_copy()
+        attempt.prepared_cost = None
+        attempt.finalized_cost = None
+        identity: Final = (
+            f"{self.litellm_call_id}-attempt-{len(self.model_call_details.get('failed_attempts_usage', ()))}"
+        )
+        params: Final = _LITELLM_PARAMS.validate_python(self.model_call_details.get("litellm_params") or {})
+        metadata: Final = StandardLoggingPayloadSetup.merge_litellm_metadata(params)
+        observed: Final[object] = self.model_call_details.get("_accounting_usage_observed")
+        cost: Final[object] = self.model_call_details.get("response_cost")
+        attempt.model_call_details = {
+            **attempt.model_call_details,
+            "litellm_call_id": identity,
+            "parent_request_id": self.litellm_call_id,
+            "_accounting_capture": None,
+            "_accounting_prior_captures": (),
+            "_accounting_usage_snapshot": UsageCaptureObservation.from_usage(
+                usage,
+                observed if isinstance(observed, UsageObservation) else UsageObservation.KNOWN,
+                float(cost) if isinstance(cost, (int, float)) else None,
+            ),
+            "litellm_params": {
+                **params,
+                "metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
+                "litellm_metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
+            },
+        }
+        captured: Final = capture(
+            attempt,
+            {"id": identity, "usage": usage.model_dump(), "status": "failed"},
+            self.start_time,
+            datetime.datetime.now(datetime.timezone.utc),
+            False,
+        )
+        if isinstance(captured, CapturedUsage):
+            self.model_call_details["_accounting_prior_captures"] = (
+                *self.model_call_details.get("_accounting_prior_captures", ()),
+                captured,
+            )
+
     def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
         """Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
         self.model_call_details["continued_partial_usage"] = usage
@@ -2796,6 +2939,55 @@ class Logging(LiteLLMLoggingBaseClass):
 
     _update_completion_start_time = update_completion_start_time
 
+    def normalize_endpoint_logging_result(self, result: object) -> object:
+        call_type: Final[object] = self.call_type
+        if call_type == CallTypes.anthropic_messages.value:
+            return self._anthropic_messages_logged_response(result=result)
+        if call_type in (CallTypes.generate_content.value, CallTypes.agenerate_content.value):
+            return self._handle_non_streaming_google_genai_generate_content_response_logging(result=result)
+        if call_type in (CallTypes.asend_message.value, CallTypes.send_message.value):
+            return self._handle_a2a_response_logging(result=result)
+        return result
+
+    def usage_capture_response(self, result: object) -> object:
+        if self.stream and not self._is_assembled_stream_success(result):
+            return None
+        copied: Final = result.model_copy(deep=True) if isinstance(result, BaseModel) else result
+        normalized: Final = self.normalize_logging_result(self.normalize_endpoint_logging_result(copied))
+        if self.model_call_details.get("call_type") == CallTypes.arealtime.value and isinstance(result, list):
+            frames: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(result)
+            measured: Final = any(
+                isinstance(_LITELLM_PARAMS.validate_python(frame.get("response") or {}).get("usage"), Mapping)
+                for frame in frames
+                if frame.get("type") == "response.done"
+            )
+            if not measured:
+                metered: Final = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
+                return {
+                    "id": self.litellm_call_id,
+                    "usage": {"metered_usage": TypeAdapter(object).dump_python(metered, mode="json")}
+                    if metered is not None
+                    else None,
+                }
+        assembled: Final = self._get_assembled_streaming_response(
+            normalized, self.start_time, self.start_time, True, []
+        )
+        return normalized if assembled is None else assembled
+
+    def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]:
+        from litellm.utils import count_characters
+
+        return TypeAdapter(dict[str, JsonValue]).validate_python(
+            {
+                "custom_pricing": self.custom_pricing_for(result),
+                "custom_cost_per_token": self.custom_cost_per_token,
+                "custom_cost_per_second": self.custom_cost_per_second,
+                "prompt_characters": count_characters(self._prompt_for_cost_calculation())
+                if self.model_call_details.get("call_type") in (CallTypes.speech.value, CallTypes.aspeech.value)
+                else None,
+            }
+        )
+
     def normalize_logging_result(self, result: object) -> object:
         """
         Some endpoints return a different type of result than what is expected by the logging system.
@@ -2806,8 +2998,20 @@ class Logging(LiteLLMLoggingBaseClass):
             combined_usage_object: Final = (
                 RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=result)
             )
+            metered: Final[object] = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
+            captured_usage: Final = (
+                combined_usage_object.model_copy(
+                    update={
+                        "metered_usage": TypeAdapter(dict[str, JsonValue]).validate_python(
+                            TypeAdapter(object).dump_python(metered, mode="json")
+                        )
+                    }
+                )
+                if metered is not None
+                else combined_usage_object
+            )
             logging_result = RealtimeAPITokenUsageProcessor.create_logging_realtime_object(
-                usage=combined_usage_object,
+                usage=captured_usage,
                 results=result,
             )
 
@@ -3068,7 +3272,14 @@ class Logging(LiteLLMLoggingBaseClass):
                 start_time = self.start_time
             if end_time is None:
                 end_time = datetime.datetime.now()
-            if self.completion_start_time is None:
+            # A request re-dispatched to a fallback reuses this logging object but runs in
+            # its own window, so a first-token time from the previous attempt cannot belong
+            # to it. Keeping it makes the attempt report a negative TTFT.
+            first_token_seconds: Final = _timestamp_seconds(self.completion_start_time)
+            attempt_start_seconds: Final = _timestamp_seconds(start_time)
+            if first_token_seconds is None or (
+                attempt_start_seconds is not None and first_token_seconds < attempt_start_seconds
+            ):
                 self.completion_start_time = end_time
                 self.model_call_details["completion_start_time"] = self.completion_start_time
 
@@ -3080,15 +3291,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 else cache_hit
             )
 
-            if self.call_type == CallTypes.anthropic_messages.value:
-                result = self._anthropic_messages_logged_response(result=result)
-            elif (
-                self.call_type == CallTypes.generate_content.value
-                or self.call_type == CallTypes.agenerate_content.value
-            ):
-                result = self._handle_non_streaming_google_genai_generate_content_response_logging(result=result)
-            elif self.call_type == CallTypes.asend_message.value or self.call_type == CallTypes.send_message.value:
-                result = self._handle_a2a_response_logging(result=result)
+            result = self.normalize_endpoint_logging_result(result)
 
             logging_result: Final = self.normalize_logging_result(result=result)
             if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
@@ -3441,6 +3644,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     self._with_client_facing_stream_model(complete_streaming_response)
                 )
                 self.model_call_details["complete_streaming_response"] = logged_streaming_response
+                self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
                 ## STANDARDIZED LOGGING PAYLOAD
                 if not for_async_request:
                     self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
@@ -3964,6 +4168,8 @@ class Logging(LiteLLMLoggingBaseClass):
                     cost_for_built_in_tools_cost_usd_dollar=0.0,
                 )
 
+        self.capture_usage_before_logging(result, start_time, end_time)
+
         self.truncated_messages_for_logging = await truncate_base64_in_messages_async(
             StandardLoggingPayloadSetup.append_system_prompt_messages(
                 kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
@@ -4990,16 +5196,34 @@ class Logging(LiteLLMLoggingBaseClass):
 
     def billable_result(self, result: object) -> object:
         """The response the cost calculator prices: a /v1/messages body served through the chat
-        bridge is replaced by the ModelResponse its spend row is built from, anything else is
+        bridge is replaced by the ModelResponse its spend row is built from, a native one is only
+        converted when its provider reports the cost the bill is read from, anything else is
         priced as is."""
         served: Final[object] = result
         call_type: Final[object] = getattr(self, "call_type", None)
         if not isinstance(result, dict):
             return served
-        if self.bridged_model_response is None and call_type != CallTypes.anthropic_messages.value:
+        if self.bridged_model_response is not None:
+            return self._anthropic_messages_logged_response(result=served)
+        if call_type != CallTypes.anthropic_messages.value:
             return served
+        if not reports_native_messages_cost(self.model_call_details.get("custom_llm_provider")):
+            return self._served_with_requested_speed(result=served)
         return self._anthropic_messages_logged_response(result=served)
 
+    def _served_with_requested_speed(self, result: object) -> object:
+        """A native /v1/messages body priced as served bills the speed the client asked for when its
+        usage block reports none, the same speed its spend row is priced at."""
+        body: Final = _SERVED_ANTHROPIC_BODY.validate_python(result)
+        usage: Final = body.get("usage")
+        requested: Final = _LITELLM_PARAMS.validate_python(getattr(self, "optional_params", None) or {}).get("speed")
+        if not isinstance(requested, str) or not isinstance(usage, dict):
+            return result
+        reported: Final = _SERVED_ANTHROPIC_BODY.validate_python(usage)
+        if isinstance(reported.get("speed"), str):
+            return result
+        return {**body, "usage": {**reported, "speed": requested}}
+
     def _anthropic_messages_logged_response(self, result: object) -> ModelResponse:
         """
         The ModelResponse a /v1/messages spend_logs row is built from.
@@ -6808,45 +7032,19 @@ class StandardLoggingPayloadSetup:
 
     @staticmethod
     def get_usage_from_response_obj(response_obj: dict | None, combined_usage_object: Usage | None = None) -> Usage:
-        ## BASE CASE ##
         if combined_usage_object is not None:
             return combined_usage_object
-        if response_obj is None:
-            return Usage(
-                prompt_tokens=0,
-                completion_tokens=0,
-                total_tokens=0,
-            )
-
-        usage: Final = response_obj.get("usage", None)
-        if usage is None:
-            rerank_usage: Final = get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj))
-            if rerank_usage is not None:
-                return Usage(**rerank_usage)
-        if usage is None or (not isinstance(usage, dict) and not isinstance(usage, Usage)):
-            return Usage(
-                prompt_tokens=0,
-                completion_tokens=0,
-                total_tokens=0,
-            )
-        elif isinstance(usage, Usage):
-            return usage
-        elif isinstance(usage, ResponseAPIUsage):
-            return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage)
-        elif isinstance(usage, dict):
-            if ResponseAPILoggingUtils.is_response_api_usage(usage):
-                return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage)
-            if InteractionsUsageObjectTransformation.is_interactions_usage_object(usage):
-                return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage)
-            return Usage(**usage)
-
-        raise ValueError(f"usage is required, got={usage} of type {type(usage)}")
+        reported: Final = response_obj.get("usage") if response_obj else None
+        if isinstance(reported, Usage):
+            return reported
+        normalized: Final = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj)
+        return Usage(**normalized) if normalized else Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
 
     @staticmethod
     def get_usage_as_dict(
-        response_obj: dict | None,
+        response_obj: dict[str, Any] | None,
         combined_usage_object: Usage | None = None,
-    ) -> dict:
+    ) -> dict[str, Any]:
         """
         Like get_usage_from_response_obj but returns a plain dict, skipping
         the Pydantic Usage construction on the hot path.
@@ -6858,7 +7056,21 @@ class StandardLoggingPayloadSetup:
             return _empty
         _raw: Final = response_obj.get("usage", None)
         if _raw is None:
-            return dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty)
+            google_usage: Final[object] = response_obj.get("usageMetadata")
+            if isinstance(google_usage, dict) and _GOOGLE_USAGE_FIELDS.intersection(google_usage):
+                return litellm.VertexGeminiConfig.calculate_usage(
+                    cast(  # cast-ok: retain converter-supported modalities and token_count aliases
+                        GenerateContentResponseBody, response_obj
+                    )
+                ).model_dump()
+            reported: Final = response_obj.get("usage_info")
+            if isinstance(reported, BaseModel):
+                return reported.model_dump(exclude_none=True)
+            return (
+                {key: value for key, value in reported.items() if value is not None}
+                if isinstance(reported, dict)
+                else dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty)
+            )
         if isinstance(_raw, ResponseAPIUsage):
             return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_raw).model_dump()
         if isinstance(_raw, dict):

litellm/litellm_core_utils/model_param_helper.py

@@ -17,6 +17,7 @@ from openai.types.responses.response_create_params import (
     ResponseCreateParamsNonStreaming,
     ResponseCreateParamsStreaming,
 )
+from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
 from litellm.types.llms.anthropic import AnthropicMessagesRequest
@@ -41,6 +42,16 @@ def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> obj
     return None
 
 
+_OPTIONAL_BOOL: Final = TypeAdapter[bool | None](bool | None)
+
+
+def embeds_thought_signature_in_tool_call_id(litellm_params: Mapping[str, object]) -> bool:
+    try:
+        return _OPTIONAL_BOOL.validate_python(litellm_params.get("thought_signature_in_tool_call_id")) is not False
+    except ValidationError:
+        return True
+
+
 class ModelParamHelper:
     # Cached at class level — deterministic set built from static OpenAI type annotations
     relevant_logging_args: ClassVar[frozenset[str]] = frozenset()

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -8,7 +8,7 @@ import io
 import json
 import mimetypes
 import re
-from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from collections.abc import Callable, Hashable, Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from graphlib import CycleError, TopologicalSorter
 from itertools import chain, groupby, islice
@@ -29,6 +29,7 @@ from litellm.types.llms.openai import (
     ChatCompletionAudioObject,
     ChatCompletionFileObject,
     ChatCompletionImageObject,
+    ChatCompletionPromptCacheBreakpoint,
     ChatCompletionReasoningItem,
     ChatCompletionReasoningSummaryTextBlock,
     ChatCompletionRedactedThinkingBlock,
@@ -238,7 +239,7 @@ def is_non_content_values_set(message: AllMessageValues) -> bool:
     return any(message.get(key, None) is not None for key in message if key not in ignore_keys)
 
 
-_IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image"})
+_IMAGE_CONTENT_PART_TYPES: Final = ("image_url", "input_image", "image", "computer_screenshot")
 _IMAGE_SCAN_MAX_DEPTH: Final = 4
 
 
@@ -557,15 +558,24 @@ def as_openai_image_part(part: Mapping[str, object]) -> ChatCompletionImageObjec
     return {"type": "image_url", "image_url": {"url": url}} if url else None
 
 
-def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool:
-    """Whether any message carries an image content part, across the dialects that reach
-    pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``,
-    and Anthropic Messages ``image``, including images nested inside ``tool_result`` blocks."""
-    return any(
-        isinstance(content, list) and _content_parts_contain_image(content)
-        for message in messages
-        for content in (message.get("content"),)
-    )
+def _message_contains_image_content(message: object) -> bool:
+    match message:
+        case {"content": list()}:
+            return _content_parts_contain_image(content_parts(_CONTENT_PART.validate_python(message)))
+        case {"type": "function_call_output" | "custom_tool_call_output" | "tool_result", "output": list()}:
+            output: Final = _CONTENT_PART.validate_python(message)["output"]
+            return _content_parts_contain_image(_CONTENT_PARTS.validate_python(output))
+        case {"type": "computer_call_output", "output": dict()}:
+            return _content_parts_contain_image((_CONTENT_PART.validate_python(message)["output"],))
+        case _:
+            return False
+
+
+def request_contains_image_content(messages: object) -> bool:
+    """Inspect native message content without translating or validating the request."""
+    if not isinstance(messages, (list, tuple)):
+        return False
+    return any(_message_contains_image_content(message) for message in _CONTENT_PARTS.validate_python(messages))
 
 
 def audio_or_image_in_message_content(message: AllMessageValues) -> bool:
@@ -2184,16 +2194,62 @@ def with_prompt_cache_breakpoint(target: _MarkedT, marker: object) -> _MarkedT:
 
 
 _PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(PromptCacheBreakpoint)
+_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(ChatCompletionPromptCacheBreakpoint)
+_AcceptedBreakpointT: Final = TypeVar("_AcceptedBreakpointT", bound=Mapping[str, object])
+
+
+def _accepted_prompt_cache_breakpoint(
+    accepted: TypeAdapter[_AcceptedBreakpointT], marker: object
+) -> _AcceptedBreakpointT | None:
+    try:
+        return accepted.validate_python(marker)
+    except ValidationError:
+        return None
+
+
+def prompt_cache_breakpoint_within(
+    accepted: TypeAdapter[_AcceptedBreakpointT], marker: object
+) -> _AcceptedBreakpointT | None:
+    """The client's marker cut down to the ``accepted`` shape: an unsupported ttl is dropped and the mode kept,
+    and a marker whose mode the target cannot read is dropped whole."""
+    whole: Final = _accepted_prompt_cache_breakpoint(accepted, marker)
+    if whole is not None:
+        return whole
+    fields: Final = _breakpoint_fields_or_none(marker)
+    without_ttl: Final = (
+        None
+        if fields is None
+        else _accepted_prompt_cache_breakpoint(accepted, {key: item for key, item in fields.items() if key != "ttl"})
+    )
+    verbose_logger.debug(
+        "Dropping malformed prompt_cache_breakpoint %r under drop_params"
+        if without_ttl is None
+        else "Dropping the unsupported ttl of prompt_cache_breakpoint %r under drop_params",
+        marker,
+    )
+    return without_ttl
 
 
 def prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
     if marker is None or not drop_params:
         return marker
-    try:
-        return _PROMPT_CACHE_BREAKPOINT.validate_python(marker)
-    except ValidationError:
-        verbose_logger.debug("Dropping malformed prompt_cache_breakpoint %r under drop_params", marker)
-        return None
+    return prompt_cache_breakpoint_within(_PROMPT_CACHE_BREAKPOINT, marker)
+
+
+def prompt_cache_breakpoint_with_readable_mode(marker: object, drop_params: bool) -> object:
+    """The client's marker as written, or None under drop_params when no target could read its mode, so the
+    target's own config decides which ttl and keys it keeps."""
+    return None if prompt_cache_breakpoint_for_wire(marker, drop_params) is None else marker
+
+
+def whole_prompt_cache_breakpoint_or_none(marker: object) -> PromptCacheBreakpoint | None:
+    return _accepted_prompt_cache_breakpoint(_PROMPT_CACHE_BREAKPOINT, marker)
+
+
+def chat_completion_prompt_cache_breakpoint_for_wire(marker: object, drop_params: bool) -> object:
+    if marker is None or not drop_params:
+        return marker
+    return prompt_cache_breakpoint_within(_CHAT_COMPLETION_PROMPT_CACHE_BREAKPOINT, marker)
 
 
 def cache_control_as_prompt_cache_breakpoint(block: _MarkedT) -> _MarkedT:
@@ -2211,6 +2267,13 @@ _BREAKPOINT_FIELDS: Final = TypeAdapter(Mapping[str, object])
 _BREAKPOINT_BLOCKS: Final = TypeAdapter(list[object])
 
 
+def _breakpoint_fields_or_none(value: object) -> Mapping[str, object] | None:
+    try:
+        return _BREAKPOINT_FIELDS.validate_python(value)
+    except ValidationError:
+        return None
+
+
 def _without_prompt_cache_breakpoint(value: object) -> object:
     if not isinstance(value, Mapping):
         return value
@@ -2233,6 +2296,27 @@ def without_prompt_cache_breakpoints(block: _MarkedT) -> _MarkedT:
     return cast(_MarkedT, stripped)  # cast-ok: same block shape as the input minus the marker keys
 
 
+def _part_with_chat_completion_prompt_cache_breakpoint(part: object) -> object:
+    if not isinstance(part, Mapping):
+        return part
+    fields: Final = _BREAKPOINT_FIELDS.validate_python(part)
+    marker: Final = fields.get("prompt_cache_breakpoint")
+    if marker is None:
+        return fields
+    unmarked: Final = {key: item for key, item in fields.items() if key != "prompt_cache_breakpoint"}
+    return with_prompt_cache_breakpoint(unmarked, chat_completion_prompt_cache_breakpoint_for_wire(marker, True))
+
+
+def chat_completion_prompt_cache_breakpoints_for_wire(message: _MarkedT, drop_params: bool) -> _MarkedT:
+    content: Final = message.get("content")
+    if not drop_params or not isinstance(content, list):
+        return message
+    parts: Final = [
+        _part_with_chat_completion_prompt_cache_breakpoint(part) for part in _BREAKPOINT_BLOCKS.validate_python(content)
+    ]
+    return cast(_MarkedT, {**message, "content": parts})  # cast-ok: same message shape with its parts' markers trimmed
+
+
 LITELLM_RESPONSE_ONLY_MESSAGE_FIELDS: Final = frozenset(
     {"thinking_blocks", "reasoning_items", "provider_specific_fields"}
 )
@@ -3226,3 +3310,15 @@ def text_completion_prompt_to_messages(prompt: object) -> tuple[AllMessageValues
     if string_entries and len(string_entries) == len(entries):
         return tuple(ChatCompletionUserMessage(role="user", content=entry) for entry in string_entries)
     raise ValueError(f"`prompt` must be a non-empty string or a non-empty list of strings. Got: {prompt_type_name}.")
+
+
+def is_present_non_string(value: object) -> bool:
+    return value is not None and not isinstance(value, str)
+
+
+def invalid_type_message(field_path: str, value: object) -> str:
+    return f"Invalid type for '{field_path}': expected a string, got {type(value).__name__}"
+
+
+def is_unhashable_value_behind(error: TypeError, value: object) -> bool:
+    return not isinstance(value, Hashable) and f"unhashable type: '{type(value).__name__}'" in str(error)

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -26,11 +26,13 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or
 from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
     anthropic_system_messages,
     find_anthropic_server_tool_result,
+    is_anthropic_server_tool_use,
     is_anthropic_tool_search_use,
     message_field,
     parts_of,
     place_mid_conversation_system,
     split_leading_system_run,
+    string_field,
 )
 from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
 from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client, header_value
@@ -1252,6 +1254,22 @@ def encode_tool_call_id_with_signature(tool_call_id: str, thought_signature: str
 _encode_tool_call_id_with_signature = encode_tool_call_id_with_signature
 
 
+def _tool_call_id_prefix(tool_call_id: object) -> str | None:
+    return tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)[0] if isinstance(tool_call_id, str) else None
+
+
+def _called_tool_name(tool_call_id: str, tool_calls: Sequence[Mapping[str, object]]) -> str | None:
+    exact_matches: Final = tuple(call for call in tool_calls if call.get("id") == tool_call_id)
+    prefix: Final = _tool_call_id_prefix(tool_call_id)
+    prefix_matches: Final = tuple(
+        call for call in tool_calls if prefix and _tool_call_id_prefix(call.get("id")) == prefix
+    )
+    matches: Final = exact_matches or (prefix_matches if len(prefix_matches) == 1 else ())
+    function: Final = matches[0].get("function") if matches else None
+    name: Final = function.get("name") if isinstance(function, Mapping) else None
+    return name if isinstance(name, str) else None
+
+
 def get_thought_signature_from_tool(tool: Mapping[str, object]) -> str | None:
     """Extract thought signature from tool call's provider_specific_fields.
 
@@ -1691,16 +1709,14 @@ def convert_to_gemini_tool_call_result(
                     )
                 ) is not None:
                     media_parts.append(media_part)
-    name: str | None = message.get("name", "")
-
     # Recover name from last message with tool calls
-    if last_message_with_tool_calls:
-        tools: Final = last_message_with_tool_calls.get("tool_calls", [])
-        msg_tool_call_id: Final = message.get("tool_call_id", None)
-        for tool in tools:
-            prev_tool_call_id = tool.get("id", None)
-            if msg_tool_call_id and prev_tool_call_id and msg_tool_call_id == prev_tool_call_id:
-                name = tool.get("function", {}).get("name", "")
+    msg_tool_call_id: Final = message.get("tool_call_id", None)
+    called_tool_name: Final = (
+        _called_tool_name(msg_tool_call_id, last_message_with_tool_calls.get("tool_calls") or ())
+        if last_message_with_tool_calls and isinstance(msg_tool_call_id, str) and msg_tool_call_id
+        else None
+    )
+    name: Final[str | None] = called_tool_name if called_tool_name is not None else message.get("name", "")
 
     # Echo the OpenAI tool_call_id on functionResponse (strip thought-signature suffix).
     gemini_call_id: str | None = None
@@ -2008,7 +2024,7 @@ def _server_tool_answers_replaced_after(messages: Sequence[object], index: int)
     following: Final = _messages_until_the_next_assistant(messages, index)
     a_user_turn_remains: Final = any(
         message_field(message, "role") in ("user", "tool", "function")
-        and message_field(message, "tool_call_id") not in kept_server_result_ids
+        and string_field(message, "tool_call_id") not in kept_server_result_ids
         for message in following
     )
     return _tool_call_ids_in_tool_messages(following) & kept_server_result_ids if a_user_turn_remains else frozenset()
@@ -2059,6 +2075,54 @@ def _placed_and_paired(
 _AnthropicToolInvokeItem: TypeAlias = AnthropicMessagesToolUseParam | dict[str, Any]
 
 
+def _is_server_tool_result(block: Mapping[str, object]) -> bool:
+    block_type: Final = block.get("type")
+    return isinstance(block_type, str) and block_type.endswith("_tool_result")
+
+
+def _split_unrun_server_tool_uses(
+    groups: Sequence[Sequence[_AnthropicToolInvokeItem]],
+) -> tuple[tuple[Sequence[_AnthropicToolInvokeItem], ...], tuple[_AnthropicToolInvokeItem, ...]]:
+    completed: Final = tuple(group for group in groups if any(_is_server_tool_result(block) for block in group))
+    unrun: Final = tuple(group[0] for group in groups if not any(_is_server_tool_result(block) for block in group))
+    return completed, unrun
+
+
+def _block_id(block: Mapping[str, object]) -> object:
+    return block.get("id")
+
+
+def _unrun_server_tool_use_ids(blocks: Sequence[Mapping[str, object]]) -> frozenset[object]:
+    result_ids: Final = frozenset(block.get("tool_use_id") for block in blocks if _is_server_tool_result(block))
+    return frozenset(
+        _block_id(block)
+        for block in blocks
+        if block.get("type") == "server_tool_use" and _block_id(block) not in result_ids
+    )
+
+
+def _last_block_id_among(blocks: Sequence[Mapping[str, object]], ids: frozenset[object]) -> object:
+    return next(block_id for block in reversed(blocks) if (block_id := _block_id(block)) in ids)
+
+
+def _paused_turn_replay(
+    tool_invokes: Sequence[_AnthropicToolInvokeItem],
+    thinking_blocks: Sequence[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock],
+    *,
+    is_final_message: bool,
+) -> tuple[
+    tuple[_AnthropicToolInvokeItem, ...],
+    tuple[ChatCompletionThinkingBlock | ChatCompletionRedactedThinkingBlock, ...],
+]:
+    unrun_ids: Final = _unrun_server_tool_use_ids(tool_invokes)
+    if not unrun_ids:
+        return tuple(tool_invokes), tuple(thinking_blocks)
+    dropped_ids: Final = (
+        unrun_ids - frozenset((_last_block_id_among(tool_invokes, unrun_ids),)) if is_final_message else unrun_ids
+    )
+    return tuple(block for block in tool_invokes if _block_id(block) not in dropped_ids), ()
+
+
 def convert_to_anthropic_tool_invoke(
     tool_calls: list[ChatCompletionAssistantToolCall],
     web_search_results: Sequence[object] | None = None,
@@ -2141,6 +2205,15 @@ def convert_to_anthropic_tool_invoke(
             anthropic_tool_invoke.append(server_tool_result)
         elif is_anthropic_tool_search_use(tool_id, tool_name) and tool_id not in answered_tool_ids:
             continue
+        elif is_anthropic_server_tool_use(tool_id) and tool_id not in answered_tool_ids:
+            anthropic_tool_invoke.append(
+                {
+                    "type": "server_tool_use",
+                    "id": tool_id,
+                    "name": tool_name,
+                    "input": tool_input,
+                }
+            )
         else:
             sanitized_tool_id = sanitize_anthropic_tool_use_id(tool_id)
             _anthropic_tool_use_param = AnthropicMessagesToolUseParam(
@@ -3109,6 +3182,30 @@ def anthropic_messages_pt(
                         _has_server_tool_calls = True
                         break
 
+            # Get web_search_results and tool_results from provider_specific_fields
+            # for server_tool_use reconstruction.
+            # Fixes: https://github.com/BerriAI/litellm/issues/17737
+            _provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
+            _provider_specific_fields: dict[str, Any] = (
+                cast(dict[str, Any], _provider_specific_fields_raw)
+                if isinstance(_provider_specific_fields_raw, dict)
+                else {}
+            )
+            tool_invoke_results = (
+                convert_to_anthropic_tool_invoke(
+                    assistant_tool_calls,
+                    web_search_results=_provider_specific_fields.get("web_search_results"),
+                    tool_results=_provider_specific_fields.get("tool_results"),
+                    answered_tool_ids=client_answered_tool_ids,
+                )
+                if assistant_tool_calls is not None
+                else ()
+            )
+            turn_paused = bool(_unrun_server_tool_use_ids(tool_invoke_results))
+            replayed_tool_invokes, replayed_thinking_blocks = _paused_turn_replay(
+                tool_invoke_results, thinking_blocks or (), is_final_message=msg_i == len(messages) - 1
+            )
+
             if (
                 thinking_blocks is not None
                 and _has_server_tool_calls
@@ -3121,26 +3218,12 @@ def anthropic_messages_pt(
                 # We must preserve this interleaved order because Anthropic
                 # verifies thinking block signatures based on position.
 
-                # Build the tool call groups (server_tool_use + its result)
-                _provider_specific_fields_raw_tc = assistant_content_block.get("provider_specific_fields")
-                _provider_specific_fields_tc: dict[str, Any] = {}
-                if isinstance(_provider_specific_fields_raw_tc, dict):
-                    _provider_specific_fields_tc = cast(dict[str, Any], _provider_specific_fields_raw_tc)
-                _web_search_results_tc = _provider_specific_fields_tc.get("web_search_results")
-                _tool_results_tc = _provider_specific_fields_tc.get("tool_results")
-                tool_invoke_results = convert_to_anthropic_tool_invoke(
-                    assistant_tool_calls,
-                    web_search_results=_web_search_results_tc,
-                    tool_results=_tool_results_tc,
-                    answered_tool_ids=client_answered_tool_ids,
-                )
-
                 # Group tool invoke results into (server_tool_use, result) pairs
                 # and separate regular tool_use blocks
                 server_tool_groups: list[list[_AnthropicToolInvokeItem]] = []
                 regular_tool_uses: list[_AnthropicToolInvokeItem] = []
                 _current_group: list[_AnthropicToolInvokeItem] = []
-                for item in tool_invoke_results:
+                for item in replayed_tool_invokes:
                     item_type = item.get("type", "") if isinstance(item, dict) else getattr(item, "type", "")
                     if item_type == "server_tool_use":
                         if _current_group:
@@ -3154,6 +3237,9 @@ def anthropic_messages_pt(
                         _current_group.append(item)
                 if _current_group:
                     server_tool_groups.append(_current_group)
+                completed_server_tool_groups, pending_server_tool_uses = _split_unrun_server_tool_uses(
+                    server_tool_groups
+                )
 
                 # Build the text block if content is a non-empty string
                 text_element = None
@@ -3177,15 +3263,15 @@ def anthropic_messages_pt(
                 # Any remaining groups (after all thinking blocks) go after.
                 tb_idx = 0
                 grp_idx = 0
-                num_tb = len(thinking_blocks) if thinking_blocks else 0
-                num_grp = len(server_tool_groups)
+                num_tb = len(replayed_thinking_blocks)
+                num_grp = len(completed_server_tool_groups)
 
                 while tb_idx < num_tb or grp_idx < num_grp:
                     if tb_idx < num_tb and grp_idx < num_grp:
                         # Emit thinking block then its tool group
-                        assistant_content.append(thinking_blocks[tb_idx])
+                        assistant_content.append(replayed_thinking_blocks[tb_idx])
                         tb_idx += 1
-                        for block in server_tool_groups[grp_idx]:
+                        for block in completed_server_tool_groups[grp_idx]:
                             item_id = block.get("id") if isinstance(block, dict) else getattr(block, "id", None)
                             if item_id and item_id in unique_tool_ids:
                                 continue
@@ -3195,11 +3281,11 @@ def anthropic_messages_pt(
                         grp_idx += 1
                     elif tb_idx < num_tb:
                         # More thinking blocks than tool groups - emit before text
-                        assistant_content.append(thinking_blocks[tb_idx])
+                        assistant_content.append(replayed_thinking_blocks[tb_idx])
                         tb_idx += 1
                     else:
                         # More tool groups than thinking blocks - emit remaining
-                        for block in server_tool_groups[grp_idx]:
+                        for block in completed_server_tool_groups[grp_idx]:
                             item_id = block.get("id") if isinstance(block, dict) else getattr(block, "id", None)
                             if item_id and item_id in unique_tool_ids:
                                 continue
@@ -3212,8 +3298,8 @@ def anthropic_messages_pt(
                 if text_element is not None:
                     assistant_content.append(text_element)
 
-                # Add regular (non-server) tool calls at the end
-                for item in regular_tool_uses:
+                # Add regular tool calls, then a paused turn's unrun server tool call, at the end
+                for item in (*regular_tool_uses, *pending_server_tool_uses):
                     item_id = item.get("id") if isinstance(item, dict) else getattr(item, "id", None)
                     if item_id and item_id in unique_tool_ids:
                         continue
@@ -3233,18 +3319,18 @@ def anthropic_messages_pt(
                 )
                 _content_list = assistant_content_block.get("content") if _content_is_list else None
 
-                if (
-                    thinking_blocks is not None and not _content_is_list
-                ):  # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
-                    assistant_content.extend(thinking_blocks)
+                if not _content_is_list:  # IMPORTANT: ADD THIS FIRST, ELSE ANTHROPIC WILL RAISE AN ERROR
+                    assistant_content.extend(replayed_thinking_blocks)
                 if _content_is_list and _content_list is not None:
-                    for m in _with_missing_thinking_blocks(_content_list, thinking_blocks or ()):
+                    for m in _with_missing_thinking_blocks(_content_list, replayed_thinking_blocks):
                         if not isinstance(m, dict):
                             continue
                         # handle thinking blocks
                         if (thinking_block := _as_thinking_block(m)) is not None:
                             assistant_content.extend(
-                                _replayable_thinking_blocks([thinking_block], drop_params=drop_params)
+                                ()
+                                if turn_paused
+                                else _replayable_thinking_blocks([thinking_block], drop_params=drop_params)
                             )
                             continue
                         text_block = cast(str, m.get("text", ""))
@@ -3284,25 +3370,9 @@ def anthropic_messages_pt(
                     assistant_content.append(_anthropic_text_content_element)
 
             if assistant_tool_calls is not None:  # support assistant tool invoke conversion
-                # Get web_search_results and tool_results from provider_specific_fields
-                # for server_tool_use reconstruction.
-                # Fixes: https://github.com/BerriAI/litellm/issues/17737
-                _provider_specific_fields_raw = assistant_content_block.get("provider_specific_fields")
-                _provider_specific_fields: dict[str, Any] = {}
-                if isinstance(_provider_specific_fields_raw, dict):
-                    _provider_specific_fields = cast(dict[str, Any], _provider_specific_fields_raw)
-                _web_search_results = _provider_specific_fields.get("web_search_results")
-                _tool_results = _provider_specific_fields.get("tool_results")
-                tool_invoke_results = convert_to_anthropic_tool_invoke(
-                    assistant_tool_calls,
-                    web_search_results=_web_search_results,
-                    tool_results=_tool_results,
-                    answered_tool_ids=client_answered_tool_ids,
-                )
-
                 # Prevent "tool_use ids must be unique" errors by filtering duplicates
                 # This can happen when merging history that already contains the tool calls
-                for item in tool_invoke_results:
+                for item in replayed_tool_invokes:
                     # tool_use items are typically dicts, but handle objects just in case
                     item_id = item.get("id") if isinstance(item, dict) else getattr(item, "id", None)
 

litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py

@@ -85,6 +85,11 @@ def message_field(message: object, key: str) -> object:
     return mapping.get(key) if mapping is not None else getattr(message, key, None)
 
 
+def string_field(message: object, key: str) -> str | None:
+    value: Final = message_field(message, key)
+    return value if isinstance(value, str) else None
+
+
 def is_system_message(message: object) -> bool:
     return message_field(message, "role") == "system"
 
@@ -286,6 +291,10 @@ def _separate_thinking_blocks_render(message: object) -> bool:
     return any(_thinking_block_renders(block) for block in parts_of(message_field(message, "thinking_blocks")))
 
 
+def is_anthropic_server_tool_use(tool_id: str) -> bool:
+    return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
+
+
 def is_anthropic_tool_search_use(tool_id: str, tool_name: str) -> bool:
     return tool_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX) and tool_name in ANTHROPIC_TOOL_SEARCH_TOOL_NAMES
 

litellm/litellm_core_utils/realtime_streaming.py

@@ -631,6 +631,8 @@ class RealTimeStreaming:
             if self.session_tools or self.tool_calls:
                 self.logging_obj.model_call_details["realtime_tools"] = self.session_tools
                 self.logging_obj.model_call_details["realtime_tool_calls"] = self.tool_calls
+            if LiteLLMLogging.usage_capture_callback is not None:
+                self.logging_obj.capture_usage_before_logging(self.messages)
             # Route through the bounded logging worker (per-coroutine timeout +
             # concurrency cap) instead of a bare create_task, so a slow callback
             # can't leave suspended tasks pinning each call's response in memory.

litellm/litellm_core_utils/streaming_chunk_builder_utils.py

@@ -1509,11 +1509,20 @@ def _delta_reasoning_items(delta: object) -> Sequence[object]:
     return cast(Sequence[object], items) if isinstance(items, list) else ()  # cast-ok: items forwarded verbatim
 
 
-def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
+def _chunk_first_delta(chunk: object) -> object:
     choices: Final = _field_of(chunk, "choices")
     if not isinstance(choices, list) or not choices:
-        return ()
-    return _delta_reasoning_items(_field_of(cast(Sequence[object], choices)[0], "delta"))  # cast-ok: untyped chunk
+        return None
+    return _field_of(cast(Sequence[object], choices)[0], "delta")  # cast-ok: untyped chunk
+
+
+def _chunk_reasoning_items(chunk: object) -> Sequence[object]:
+    return _delta_reasoning_items(_chunk_first_delta(chunk))
+
+
+def _chunk_refusal(chunk: object) -> str:
+    refusal: Final = _field_of(_chunk_first_delta(chunk), "refusal")
+    return refusal if isinstance(refusal, str) else ""
 
 
 _NON_TEXT_DELTA_FIELDS: Final = (
@@ -1526,6 +1535,7 @@ _NON_TEXT_DELTA_FIELDS: Final = (
     "audio",
     "images",
     "provider_specific_fields",
+    "refusal",
 )
 
 
@@ -1674,6 +1684,9 @@ def _assembled_message_fields(processor: ChunkProcessor, chunks: Sequence) -> It
         yield "reasoning_content", processor.get_combined_reasoning_content(reasoning_chunks)
 
     typed_chunks: Final = cast(Sequence[object], chunks)  # cast-ok: chunks are untyped upstream
+    combined_refusal: Final = "".join(_chunk_refusal(chunk) for chunk in typed_chunks)
+    if combined_refusal:
+        yield "refusal", combined_refusal
     reasoning_items: Final = tuple(chain.from_iterable(_chunk_reasoning_items(chunk) for chunk in typed_chunks))
     if reasoning_items:
         yield "reasoning_items", list(reasoning_items)

litellm/litellm_core_utils/streaming_handler.py

@@ -349,11 +349,14 @@ class CustomStreamWrapper:
         make_call: Callable | None = None,
         _response_headers: dict | httpx.Headers | None = None,
         count_prompt_tokens: Callable[[], int] | None = None,
+        is_mock: bool = False,
     ):
         self.model = model
         self.make_call = make_call
         self.count_prompt_tokens = count_prompt_tokens
         self.custom_llm_provider = custom_llm_provider
+        self.is_mock = is_mock
+        self.uses_openai_chat_stream = False
         self.logging_obj: LiteLLMLoggingObject = logging_obj
         self.completion_stream = completion_stream
         self.sent_first_chunk = False
@@ -376,6 +379,7 @@ class CustomStreamWrapper:
         self._upstream_provider: str | None = None
         self.received_finish_reason: str | None = None
         self.intermittent_finish_reason: str | None = None  # finish reasons that show up mid-stream
+        self.received_any_finish_reason = False  # a later chunk can reset intermittent_finish_reason
         self.special_tokens = [
             "<|assistant|>",
             "<|system|>",
@@ -901,13 +905,21 @@ class CustomStreamWrapper:
         except Exception as e:
             raise e
 
+    def _served_openrouter_model(self) -> str | None:
+        if self.custom_llm_provider == LlmProviders.OPENROUTER.value or (
+            self.custom_llm_provider == "cached_response"
+            and self.logging_obj.model_call_details.get("custom_llm_provider") == LlmProviders.OPENROUTER.value
+        ):
+            return self._provider_response_model
+        return None
+
     def model_response_creator(
         self,
         chunk: dict | None = None,
         hidden_params: Mapping[str, object] | None = None,
         delta: Delta | None = None,
     ) -> ModelResponseStream:
-        _model: Final = self._cached_model_name
+        _model: Final = self._served_openrouter_model() or self._cached_model_name
         _logging_obj_llm_provider: Final = self._cached_logging_llm_provider
 
         if chunk is None:
@@ -1302,8 +1314,8 @@ class CustomStreamWrapper:
             if _is_delta_empty:
                 model_response.choices[0].delta = Delta(content=self._close_open_think_block())
                 # get any function call arguments
-                model_response.choices[0].finish_reason = map_finish_reason(
-                    finish_reason=self.received_finish_reason
+                model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
+                    map_finish_reason(finish_reason=self.received_finish_reason)
                 )  # ensure consistent output to openai
 
                 self.sent_last_chunk = True
@@ -1597,6 +1609,7 @@ class CustomStreamWrapper:
             if response_obj["is_finished"]:
                 self.received_finish_reason = response_obj["finish_reason"]
         else:  # openai / azure chat model
+            self.uses_openai_chat_stream = True
             if isinstance(chunk, ChatCompletionChunk):
                 self._openai_handler_receives_chat_chunks = True
                 if any(str(choice.finish_reason) == "abort" for choice in chunk.choices or ()):
@@ -1617,6 +1630,7 @@ class CustomStreamWrapper:
                 return _ProviderChunkEarlyReturn(None)
             completion_obj["content"] = response_obj["text"]
             self.intermittent_finish_reason = response_obj.get("finish_reason", None)
+            self.received_any_finish_reason |= response_obj["finish_reason"] is not None
             if response_obj["is_finished"]:
                 if response_obj["finish_reason"] == "error":
                     raise Exception(
@@ -1693,7 +1707,7 @@ class CustomStreamWrapper:
                 return dispatch_result.value
             response_obj = dispatch_result.response_obj
 
-            model_response.model = self.model
+            model_response.model = self._served_openrouter_model() or self.model
             ## FUNCTION CALL PARSING
             original_chunk: Final[_UpstreamChunkLike | None] = (
                 response_obj.get("original_chunk") if response_obj is not None else None
@@ -1737,6 +1751,8 @@ class CustomStreamWrapper:
                                         # if function returned but type set to None - mistral's api returns type: None
                                         tool["type"] = "function"
                             model_response.choices[0].delta = Delta(**_json_delta)
+                            if model_response.choices[0].delta.tool_calls:
+                                self.tool_call = True
                         except Exception as e:
                             verbose_logger.exception(
                                 "litellm.CustomStreamWrapper.chunk_creator(): Exception occured - %s", e
@@ -1966,6 +1982,11 @@ class CustomStreamWrapper:
         if self.logging_obj.is_sync_litellm_request(litellm_params):
             self.logging_obj.success_handler(processed_chunk, None, None, cache_hit)
 
+    def _normalize_tool_call_finish_reason(self, finish_reason: str) -> str:
+        if finish_reason == "stop" and self.tool_call:
+            return "tool_calls"
+        return finish_reason
+
     def finish_reason_handler(self):
         model_response: Final = self.model_response_creator(
             hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id}
@@ -1981,21 +2002,30 @@ class CustomStreamWrapper:
         if served_tier is not None:
             setattr(model_response, "service_tier", served_tier)  # noqa: B010  # pydantic extra, not a declared field
         _finish_reason: Final = self.received_finish_reason or self.intermittent_finish_reason
-        if _finish_reason is not None:
-            model_response.choices[0].finish_reason = _finish_reason
-        else:
-            model_response.choices[0].finish_reason = "stop"
-
-        ## if tool use
-        if (
-            model_response.choices[0].finish_reason == "stop" and self.tool_call
-        ):  # don't overwrite for other - potential error finish reasons
-            model_response.choices[0].finish_reason = "tool_calls"
+        model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
+            _finish_reason if _finish_reason is not None else "stop"
+        )
         close_tag: Final = self._close_open_think_block()
         if close_tag is not None:
             model_response.choices[0].delta.content = close_tag
         return model_response
 
+    def _should_fail_on_clean_eof(self) -> bool:
+        return (
+            not self.is_mock
+            and self.uses_openai_chat_stream
+            and self.received_finish_reason is None
+            and self.intermittent_finish_reason is None
+            and not self.received_any_finish_reason
+        )
+
+    @staticmethod
+    def _stream_eof_error() -> httpx.RemoteProtocolError:
+        return httpx.RemoteProtocolError(
+            "Stream ended before a finish_reason was received",
+            request=httpx.Request(method="POST", url="https://docs.litellm.ai/docs"),
+        )
+
     def _final_chunk_for_logging(self, processed_chunk: "ModelResponseStream") -> "ModelResponseStream | None":
         if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
             return processed_chunk
@@ -2181,6 +2211,13 @@ class CustomStreamWrapper:
                     return response
 
         except StopIteration:
+            if self._should_fail_on_clean_eof():
+                error: Final = self._stream_eof_error()
+                eof_traceback: Final = traceback.format_exc()
+                self._record_partial_usage_for_failure()
+                threading.Thread(target=self.logging_obj.failure_handler, args=(error, eof_traceback)).start()
+                self.close()
+                self._handle_stream_fallback_error(error)
             if self.sent_last_chunk is True:
                 complete_streaming_response: Final = self._build_complete_streaming_response()
                 response = self._new_usage_chunk()
@@ -2384,6 +2421,9 @@ class CustomStreamWrapper:
                             continue
                         return usage_free_chunk
         except (StopAsyncIteration, StopIteration):
+            if self._should_fail_on_clean_eof():
+                await self.aclose()
+                self._log_stream_failure_and_raise(self._stream_eof_error())
             return await self._finalize_completed_stream(cache_hit=cache_hit)
         except httpx.TimeoutException as e:  # if httpx read timeout error occues
             traceback_exception = traceback.format_exc()
@@ -2529,7 +2569,7 @@ class CustomStreamWrapper:
         if partial is None or not isinstance(usage, Usage):
             return None
         if self.model:
-            partial.model = self.model
+            partial.model = self._served_openrouter_model() or self.model
         backfill_missing_cache_usage_fields(usage)
         return partial
 

litellm/llms/a2a/common_utils.py

@@ -160,8 +160,11 @@ async def resolve_a2a_hop_auth_header(
     litellm_params: Mapping[str, object],
     custom_llm_provider: object,
     resolve_entra_header: AgentAuthHeaderResolver = resolve_azure_ai_agent_auth_header,
+    agent_extra_headers: Mapping[str, str] | None = None,
 ) -> Mapping[str, str] | None:
-    """Entra credentials authenticate the A2A hop only; a completion-bridge agent hands them to the model provider it bridges to."""
-    if not a2a_hop_uses_entra(litellm_params, custom_llm_provider):
+    if a2a_hop_uses_entra(litellm_params, custom_llm_provider):
+        return await resolve_entra_header(litellm_params)
+    if custom_llm_provider or any(name.lower() == "authorization" for name in (agent_extra_headers or {})):
         return None
-    return await resolve_entra_header(litellm_params)
+    api_key: Final = litellm_params.get("api_key")
+    return {"Authorization": f"Bearer {api_key}"} if isinstance(api_key, str) and api_key else None

litellm/llms/anthropic/chat/handler.py

@@ -70,7 +70,12 @@ from litellm.types.utils import (
 )
 
 from ...base import BaseLLM
-from ..common_utils import AnthropicError, process_anthropic_headers, refusal_provider_specific_fields
+from ..common_utils import (
+    AnthropicError,
+    process_anthropic_headers,
+    refusal_params,
+    stop_reason_provider_specific_fields,
+)
 from .transformation import ANTHROPIC_TOOL_NAME_REVERSE_MAP_KEY, AnthropicConfig, JsonModeRequest
 
 if TYPE_CHECKING:
@@ -926,6 +931,17 @@ class ModelResponseIterator:
     def _web_search_call_snapshot(self) -> dict[str, object]:
         return dict(self._web_search_calls)
 
+    def _refresh_pending_web_search_call(self, tool_use_id: str) -> bool:
+        if tool_use_id not in self._web_search_calls:
+            return False
+        self._web_search_calls[tool_use_id] = build_web_search_call(
+            tool_use_id,
+            self._server_tool_inputs.get(tool_use_id, {}),
+            {"content": []},
+            status="in_progress",
+        )
+        return True
+
     def _complete_web_search_call(self, result: dict[str, object]) -> None:
         tool_use_id: Final = result.get("tool_use_id")
         if not isinstance(tool_use_id, str) or tool_use_id not in self._web_search_calls:
@@ -1128,6 +1144,8 @@ class ModelResponseIterator:
                                 self._server_tool_inputs[self._current_server_tool_id] = json.loads(args)
                             except (json.JSONDecodeError, TypeError):
                                 pass
+                        if self._refresh_pending_web_search_call(self._current_server_tool_id):
+                            provider_specific_fields["web_search_calls"] = self._web_search_call_snapshot()
                         self._current_server_tool_id = None
                 if self.is_response_format_tool:
                     text = self._flush_response_format_buffer()
@@ -1140,8 +1158,8 @@ class ModelResponseIterator:
                 # These are automatically handled by Anthropic API, we just pass them through
                 pass
             elif type_chunk == "message_delta":
-                finish_reason, usage, container, refusal_fields = self._handle_message_delta(chunk)
-                provider_specific_fields.update(refusal_fields)
+                finish_reason, usage, container, stop_reason_fields = self._handle_message_delta(chunk)
+                provider_specific_fields.update(stop_reason_fields)
                 text = self._release_answer_close() + self._drain_split_answer_blocks()
                 if container:
                     provider_specific_fields["container"] = container
@@ -1190,6 +1208,7 @@ class ModelResponseIterator:
                 )
 
             text, tool_use = self._handle_json_mode_chunk(text=text, tool_use=tool_use)
+            refusal_fields: Final = refusal_params(provider_specific_fields)
 
             returned_chunk: Final = ModelResponseStream(
                 choices=[
@@ -1201,6 +1220,7 @@ class ModelResponseIterator:
                             provider_specific_fields=(provider_specific_fields if provider_specific_fields else None),
                             thinking_blocks=(thinking_blocks if thinking_blocks else None),
                             reasoning_content=reasoning_content,
+                            **refusal_fields,
                         ),
                         finish_reason=finish_reason,
                     )
@@ -1371,7 +1391,7 @@ class ModelResponseIterator:
             self._handle_usage(anthropic_usage_chunk=message_delta["usage"]) if "usage" in message_delta else None
         )
         container: Final = message_delta["delta"].get("container")
-        return finish_reason, usage, container, refusal_provider_specific_fields(message_delta["delta"])
+        return finish_reason, usage, container, stop_reason_provider_specific_fields(message_delta["delta"])
 
     def _handle_accumulated_json_chunk(self, data_str: str, is_final: bool = False) -> ModelResponseStream | None:
         """

litellm/llms/anthropic/chat/transformation.py

@@ -2,7 +2,9 @@ import copy
 import json
 import re
 import time
-from collections.abc import Callable, Mapping, Sequence
+from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
+from functools import partial
+from itertools import chain
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, ClassVar, Final, NoReturn, cast
 
@@ -27,7 +29,10 @@ from litellm.constants import (
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, map_finish_reason
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     DEFS_MAX_INLINED_BYTES,
+    invalid_type_message,
     is_object_schema,
+    is_present_non_string,
+    is_unhashable_value_behind,
     sanitize_input_schema_for_anthropic,
     schema_fragment_ref,
     schema_grammar_branch_count,
@@ -45,6 +50,8 @@ from litellm.litellm_core_utils.prompt_templates.image_handling import (
     inline_remote_image_urls,
 )
 from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import (
+    message_field,
+    parts_of,
     place_mid_conversation_system,
     split_leading_system_run,
 )
@@ -120,13 +127,14 @@ from litellm.utils import (
 )
 
 from ..common_utils import (
-    REFUSAL_STOP_FIELDS,
+    STOP_REASON_FIELDS,
     AnthropicError,
     AnthropicModelInfo,
     eager_input_streaming_flag,
     process_anthropic_headers,
-    refusal_provider_specific_fields,
+    refusal_params,
     requires_native_compaction_beta,
+    stop_reason_provider_specific_fields,
     strip_advisor_blocks_from_messages,
     tools_marked_for_a_tool_config_point,
 )
@@ -167,6 +175,8 @@ _DEFER_LOADING_ADAPTER: Final = TypeAdapter(bool)
 _TOOL_FIELDS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 _TOOLS_ADAPTER: Final = TypeAdapter(list[object])
 _LITELLM_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+_RESPONSE_BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
+_PENDING_WEB_SEARCH_RESULT: Final = MappingProxyType({"content": ()})
 _REQUEST_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 _NATIVE_OUTPUT_STRING_FORMATS: Final = frozenset(
     ("date-time", "time", "date", "duration", "email", "hostname", "uri", "ipv4", "ipv6", "uuid")
@@ -176,6 +186,76 @@ NATIVE_OUTPUT_MAX_OPTIONAL_PARAMETERS: Final = 24
 NATIVE_OUTPUT_MAX_GRAMMAR_BRANCHES: Final = 2048
 
 
+def _tool_call_ids_failing(
+    index: int, tool_calls: object, fails: Callable[[object], bool]
+) -> Iterator[tuple[str, object]]:
+    ids: Final = ((position, message_field(tool_call, "id")) for position, tool_call in enumerate(parts_of(tool_calls)))
+    return (
+        (f"messages[{index}].tool_calls[{position}].id", tool_call_id)
+        for position, tool_call_id in ids
+        if fails(tool_call_id)
+    )
+
+
+def _roles_failing(messages: Sequence[object], fails: Callable[[object], bool]) -> Iterator[tuple[str, object]]:
+    roles: Final = ((index, message_field(message, "role")) for index, message in enumerate(messages))
+    return ((f"messages[{index}].role", role) for index, role in roles if fails(role))
+
+
+def _is_non_empty_non_string(value: object) -> bool:
+    return bool(value) and is_present_non_string(value)
+
+
+def _assistant_tool_call_ids_failing(
+    messages: Sequence[object], fails: Callable[[object], bool]
+) -> Iterator[tuple[str, object]]:
+    assistant_turns: Final = (
+        (index, message) for index, message in enumerate(messages) if message_field(message, "role") == "assistant"
+    )
+    return chain.from_iterable(
+        _tool_call_ids_failing(index, message_field(message, "tool_calls"), fails) for index, message in assistant_turns
+    )
+
+
+def _tool_answer_ids_failing(
+    messages: Sequence[object], fails: Callable[[object], bool]
+) -> Iterator[tuple[str, object]]:
+    answers: Final = (
+        (index, message_field(message, "tool_call_id"))
+        for index, message in enumerate(messages)
+        if message_field(message, "role") in ("tool", "function")
+    )
+    return ((f"messages[{index}].tool_call_id", tool_call_id) for index, tool_call_id in answers if fails(tool_call_id))
+
+
+def _is_non_string_value_behind(error: AttributeError, value: object) -> bool:
+    return is_present_non_string(value) and value is error.obj
+
+
+def _fields_behind(messages: Sequence[object], error: Exception) -> Iterable[tuple[str, object]]:
+    match error:
+        case TypeError():
+            caused_error: Final = partial(is_unhashable_value_behind, error)
+            return chain(
+                _assistant_tool_call_ids_failing(messages, caused_error),
+                _tool_answer_ids_failing(messages, caused_error),
+                _roles_failing(messages, caused_error),
+            )
+        case AttributeError():
+            return _assistant_tool_call_ids_failing(messages, partial(_is_non_string_value_behind, error))
+        case _:
+            return ()
+
+
+def non_string_message_field_error(messages: Sequence[object], error: Exception) -> AnthropicError | None:
+    malformed_field: Final = next(
+        chain(_roles_failing(messages, _is_non_empty_non_string), _fields_behind(messages, error)), None
+    )
+    if malformed_field is None:
+        return None
+    return AnthropicError(status_code=400, message=invalid_type_message(*malformed_field))
+
+
 def _tool_type(tool: object) -> str | None:
     match tool:
         case {"type": str() as tool_type}:
@@ -2367,12 +2447,18 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         # Handling anthropic API Prompt Caching
         if len(anthropic_system_message_list) > 0:
             optional_params["system"] = anthropic_system_message_list
-        conversation: Final = place_mid_conversation_system(
-            later_messages,
-            supports_mid_conversation_system=supports_mid_conversation_system(
-                model=model, custom_llm_provider=self.custom_llm_provider
-            ),
-        )
+        try:
+            conversation: Final = place_mid_conversation_system(
+                later_messages,
+                supports_mid_conversation_system=supports_mid_conversation_system(
+                    model=model, custom_llm_provider=self.custom_llm_provider
+                ),
+            )
+        except TypeError as error:
+            field_error: Final = non_string_message_field_error(messages, error)
+            if field_error is None:
+                raise
+            raise field_error from error
         # Format rest of message according to anthropic guidelines
         try:
             anthropic_messages = anthropic_messages_pt(
@@ -2382,7 +2468,8 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 drop_params=drop_params,
             )
         except Exception as e:
-            raise AnthropicError(
+            conversion_field_error: Final = non_string_message_field_error(messages, e)
+            raise conversion_field_error or AnthropicError(
                 status_code=400,
                 message=f"{e}\nReceived Messages={request_messages}",
             )  # don't use verbose_logger.exception, if exception is raised
@@ -2919,28 +3006,24 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         web_search_results: Sequence[object],
         completion_response: Mapping[str, object],
     ) -> list[object]:
-        content: Final = completion_response.get("content")
-        blocks: Final = content if isinstance(content, Sequence) else ()
-        inputs: Final = {
-            call_id: tool_input
-            for block in blocks
-            if isinstance(block, Mapping)
-            and block.get("type") == "server_tool_use"
-            and block.get("name") == "web_search"
-            and isinstance((call_id := block.get("id")), str)
-            and isinstance((tool_input := block.get("input")), Mapping)
+        blocks: Final = _RESPONSE_BLOCKS.validate_python(completion_response.get("content") or ())
+        results_by_id: Final = {
+            tool_use_id: result
+            for result in _RESPONSE_BLOCKS.validate_python(web_search_results)
+            if result.get("type") == "web_search_tool_result"
+            and isinstance((tool_use_id := result.get("tool_use_id")), str)
         }
         return [
             build_web_search_call(
-                tool_id=tool_use_id,
-                tool_input=inputs.get(tool_use_id, {}),
-                result=result,
+                tool_id=call_id,
+                tool_input=block.get("input"),
+                result=results_by_id.get(call_id, _PENDING_WEB_SEARCH_RESULT),
+                status=None if call_id in results_by_id else "in_progress",
             )
-            for result in web_search_results
-            if isinstance(result, dict)
-            and result.get("type") == "web_search_tool_result"
-            and isinstance((tool_use_id := result.get("tool_use_id")), str)
-            and tool_use_id in inputs
+            for block in blocks
+            if block.get("type") == "server_tool_use"
+            and block.get("name") == "web_search"
+            and isinstance((call_id := block.get("id")), str)
         ]
 
     def _build_provider_specific_fields(
@@ -2964,10 +3047,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
 
         if web_search_results is not None:
             provider_specific_fields["web_search_results"] = web_search_results
-            provider_specific_fields["web_search_calls"] = self._build_web_search_calls(
-                web_search_results,
-                completion_response,
-            )
+
+        web_search_calls: Final = self._build_web_search_calls(web_search_results or (), completion_response)
+        if web_search_results is not None or web_search_calls:
+            provider_specific_fields["web_search_calls"] = web_search_calls
 
         if tool_results is not None:
             provider_specific_fields["tool_results"] = tool_results
@@ -2989,10 +3072,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         if compaction_blocks is not None:
             provider_specific_fields["compaction_blocks"] = compaction_blocks
 
-        refusal_fields: Final = refusal_provider_specific_fields(
-            REFUSAL_STOP_FIELDS.validate_python(completion_response)
+        stop_reason_fields: Final = stop_reason_provider_specific_fields(
+            STOP_REASON_FIELDS.validate_python(completion_response)
         )
-        return {**provider_specific_fields, **refusal_fields}
+        return {**provider_specific_fields, **stop_reason_fields}
 
     def transform_parsed_response(
         self,
@@ -3067,22 +3150,26 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
         if json_extra_content:
             merged_text = merged_text + json_extra_content if merged_text else json_extra_content
 
+        refusal_fields: Final = refusal_params(STOP_REASON_FIELDS.validate_python(completion_response))
         _message = litellm.Message(
             tool_calls=tool_calls_for_message,
             content=merged_text or None,
             provider_specific_fields=provider_specific_fields,
             thinking_blocks=thinking_blocks,
             reasoning_content=reasoning_content,
+            **refusal_fields,
         )
         _message.provider_specific_fields = provider_specific_fields
 
         if json_mode_message is not None:
-            refusal_fields: Final = refusal_provider_specific_fields(
-                REFUSAL_STOP_FIELDS.validate_python(completion_response)
+            stop_reason_fields: Final = stop_reason_provider_specific_fields(
+                STOP_REASON_FIELDS.validate_python(completion_response)
             )
             completion_response["stop_reason"] = "stop"
-            if refusal_fields:
-                json_mode_message.provider_specific_fields = dict(refusal_fields)
+            if stop_reason_fields:
+                json_mode_message.provider_specific_fields = dict(stop_reason_fields)
+            if "refusal" in refusal_fields:
+                json_mode_message.refusal = refusal_fields["refusal"]
             if thinking_blocks:
                 json_mode_message.thinking_blocks = thinking_blocks
                 json_mode_message.reasoning_content = reasoning_content

litellm/llms/anthropic/common_utils.py

@@ -1297,7 +1297,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             return None
         return anthropic_beta_header.split(",")
 
-    def get_computer_tool_beta_header(self, computer_tool_version: str) -> str:
+    def get_computer_tool_beta_header(self, computer_tool_version: str) -> str | None:
         """
         Get the appropriate beta header for a given computer tool version.
 
@@ -1305,16 +1305,14 @@ class AnthropicModelInfo(BaseLLMModelInfo):
             computer_tool_version: The computer tool version (e.g., 'computer_20250124', 'computer_20241022')
 
         Returns:
-            The corresponding beta header string
+            The corresponding beta header string, or None for a version with no known beta
         """
         computer_tool_beta_mapping: Final = {
+            "computer_20251124": "computer-use-2025-11-24",
             "computer_20250124": "computer-use-2025-01-24",
             "computer_20241022": "computer-use-2024-10-22",
         }
-        return computer_tool_beta_mapping.get(
-            computer_tool_version,
-            "computer-use-2024-10-22",  # Default fallback
-        )
+        return computer_tool_beta_mapping.get(computer_tool_version)
 
     def get_anthropic_beta_list(
         self,
@@ -1346,9 +1344,8 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         if effort_used:
             betas.append(ANTHROPIC_EFFORT_BETA_HEADER)  # effort-2025-11-24
 
-        if computer_tool_used:
-            beta_header: Final = self.get_computer_tool_beta_header(computer_tool_used)
-            betas.append(beta_header)
+        if computer_tool_used and (computer_tool_beta := self.get_computer_tool_beta_header(computer_tool_used)):
+            betas.append(computer_tool_beta)
 
         # Anthropic no longer requires the prompt-caching beta header
         # Prompt caching now works automatically when cache_control is used in messages
@@ -1437,9 +1434,8 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         # Anthropic no longer requires the prompt-caching beta header
         # Prompt caching now works automatically when cache_control is used in messages
         # Reference: https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
-        if computer_tool_used:
-            beta_header: Final = self.get_computer_tool_beta_header(computer_tool_used)
-            betas.add(beta_header)
+        if computer_tool_used and (computer_tool_beta := self.get_computer_tool_beta_header(computer_tool_used)):
+            betas.add(computer_tool_beta)
         # if pdf_used:
         #     betas.add("pdfs-2024-09-25")
         if file_id_used:
@@ -2518,7 +2514,7 @@ def create_anthropic_model_list_response(
     }
 
 
-class RefusalStopFields(TypedDict, total=False):
+class StopReasonFields(TypedDict, total=False):
     stop_reason: ReadOnly[object]
     stop_details: ReadOnly[object]
 
@@ -2535,16 +2531,38 @@ def _dict_or_empty(value: object) -> object:
     return value if _is_dict(value) else _no_stop_fields()
 
 
-REFUSAL_STOP_FIELDS: Final[TypeAdapter[RefusalStopFields]] = TypeAdapter(
-    Annotated[RefusalStopFields, BeforeValidator(_dict_or_empty)]
+STOP_REASON_FIELDS: Final[TypeAdapter[StopReasonFields]] = TypeAdapter(
+    Annotated[StopReasonFields, BeforeValidator(_dict_or_empty)]
 )
 
+_STOP_REASONS_KEPT_RAW: Final = frozenset({"refusal", "pause_turn"})
 
-def refusal_provider_specific_fields(message: Mapping[str, object]) -> Mapping[str, object]:
+
+def stop_reason_provider_specific_fields(message: Mapping[str, object]) -> Mapping[str, object]:
     stop_reason: Final = message.get("stop_reason")
     stop_details: Final = message.get("stop_details")
     if stop_details is not None:
         return {"stop_reason": stop_reason, "stop_details": stop_details}
-    if stop_reason == "refusal":
+    if stop_reason in _STOP_REASONS_KEPT_RAW:
         return {"stop_reason": stop_reason}
     return {}
+
+
+class _RefusalStopDetails(TypedDict, total=False):
+    explanation: ReadOnly[object]
+
+
+_REFUSAL_STOP_DETAILS: Final[TypeAdapter[_RefusalStopDetails]] = TypeAdapter(
+    Annotated[_RefusalStopDetails, BeforeValidator(_dict_or_empty)]
+)
+
+
+class RefusalParams(TypedDict, total=False):
+    refusal: ReadOnly[str]
+
+
+def refusal_params(message: Mapping[str, object]) -> RefusalParams:
+    if message.get("stop_reason") != "refusal":
+        return {}
+    explanation: Final = _REFUSAL_STOP_DETAILS.validate_python(message.get("stop_details")).get("explanation")
+    return {"refusal": explanation} if isinstance(explanation, str) and explanation else {}

litellm/llms/anthropic/messages/adapters/handler.py

@@ -118,9 +118,10 @@ def _extract_proxy_litellm_metadata(
 
 def _requested_router(kwargs: Mapping[str, object]) -> "Router | None":
     from litellm.router import Router
+    from litellm.router_python_core import PythonCore
 
     requested: Final = kwargs.get("litellm_router")
-    return requested if isinstance(requested, Router) else None
+    return requested if isinstance(requested, (Router, PythonCore)) else None  # pyright: ignore[reportReturnType]  # the router passes its core, which carries Router's members
 
 
 def request_safeguards_evaluator(

litellm/llms/anthropic/messages/adapters/streaming_iterator.py

@@ -33,6 +33,7 @@ from litellm.llms.anthropic.messages.safeguards import StreamedSafeguardResults
 from litellm.llms.anthropic.messages.utils import anthropic_sse_frame
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.llms.anthropic import (
+    AnthropicStopDetails,
     AppliedEdit,
     CompactionBlock,
     ContentBlockDelta,
@@ -284,9 +285,7 @@ class _CombinedChunkSplitter:
     @staticmethod
     def _is_combined(chunk: "ModelResponseStream") -> bool:
         """True if ``chunk`` carries response content AND a finish_reason."""
-        from litellm.llms.anthropic.messages.native.utils import (
-            openai_chat_refusal_text,
-        )
+        from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
 
         choices: Final = _optional_attr_sequence(chunk, "choices")
         if not choices:
@@ -302,7 +301,7 @@ class _CombinedChunkSplitter:
             or _optional_attr(delta, "tool_calls")
             or _optional_attr(delta, "reasoning_content")
             or _optional_attr(delta, "thinking_blocks")
-            or openai_chat_refusal_text(delta)
+            or refusal_reply_text(delta)
         )
 
     _PAYLOAD_FIELD_GROUPS: "tuple[tuple[str, ...], ...]" = (
@@ -659,6 +658,8 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         self.compaction_block = compaction_block
         self.iterations_usage = iterations_usage
         self._refusal_text: str = ""
+        self._paused_turn: bool = False
+        self._claude_refusal_stop_details: AnthropicStopDetails | None = None
         self.sent_compaction_block: bool = False
         # Per-phase flags so the compaction block's start/delta/stop events
         # are emitted (and the public state machine is advanced) in
@@ -953,7 +954,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     current_content_block_index=self.current_content_block_index,
                     applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
                 )
-                processed_chunk = self._with_refusal_stop_details(processed_chunk)
+                processed_chunk = self._with_stop_reason_fields(processed_chunk)
 
                 # Check if this is a usage chunk and we have a held stop_reason chunk
                 if will_merge_into_held:
@@ -1188,7 +1189,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
                     current_content_block_index=self.current_content_block_index,
                     applied_edits=(self.applied_edits if is_final_chunk and not will_merge_into_held else None),
                 )
-                processed_chunk = self._with_refusal_stop_details(processed_chunk)
+                processed_chunk = self._with_stop_reason_fields(processed_chunk)
 
                 # Check if this is a usage chunk and we have a held stop_reason chunk
                 if will_merge_into_held:
@@ -1406,14 +1407,22 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     def _increment_content_block_index(self):
         self.current_content_block_index += 1
 
-    def _with_refusal_stop_details(
+    def _with_stop_reason_fields(
         self,
         processed_chunk: ContentBlockDelta | MessageBlockDelta,
     ) -> ContentBlockDelta | MessageBlockDelta:
-        if processed_chunk["type"] != "message_delta" or not self._refusal_text:
+        if processed_chunk["type"] != "message_delta":
             return processed_chunk
         delta: Final = processed_chunk["delta"]
-        if delta.get("stop_reason") == "max_tokens":
+        if self._paused_turn:
+            paused_delta: Final[MessageDelta] = {**delta, "stop_reason": "pause_turn"}
+            paused_chunk: Final[MessageBlockDelta] = {**processed_chunk, "delta": paused_delta}
+            return paused_chunk
+        if delta.get("stop_reason") == "refusal" and self._claude_refusal_stop_details is not None:
+            claude_refusal_delta: Final[MessageDelta] = {**delta, "stop_details": self._claude_refusal_stop_details}
+            claude_refusal_chunk: Final[MessageBlockDelta] = {**processed_chunk, "delta": claude_refusal_delta}
+            return claude_refusal_chunk
+        if not self._refusal_text or delta.get("stop_reason") == "max_tokens":
             return processed_chunk
         from litellm.llms.anthropic.messages.native.utils import (
             refusal_stop_details,
@@ -1470,9 +1479,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
     @staticmethod
     def _is_blank_delta(chunk: "ModelResponseStream") -> bool:
         from litellm.llms.anthropic.common_utils import is_empty_unsigned_thinking_block
-        from litellm.llms.anthropic.messages.native.utils import (
-            openai_chat_refusal_text,
-        )
+        from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
 
         choice: Final = chunk.choices[0]
         if choice.finish_reason is not None:
@@ -1482,7 +1489,7 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
             return False
         if getattr(delta, "content", None):
             return False
-        if openai_chat_refusal_text(delta):
+        if refusal_reply_text(delta):
             return False
         if getattr(delta, "reasoning_content", None):
             return False
@@ -1497,6 +1504,18 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
             return False
         return True
 
+    def _record_anthropic_stop_fields(self, delta: object) -> None:
+        from litellm.llms.anthropic.messages.native.utils import (
+            anthropic_refusal_stop_details,
+            anthropic_stop_reason,
+        )
+
+        if anthropic_stop_reason(delta) == "pause_turn":
+            self._paused_turn = True
+        claude_refusal_stop_details: Final = anthropic_refusal_stop_details(delta)
+        if claude_refusal_stop_details is not None:
+            self._claude_refusal_stop_details = claude_refusal_stop_details
+
     def _should_start_new_content_block(self, chunk: "ModelResponseStream") -> bool:
         """
         Determine if we should start a new content block based on the processed chunk.
@@ -1507,16 +1526,15 @@ class AnthropicStreamWrapper(AdapterCompletionStreamWrapper):
         - Different content types in the response
         - Specific markers in the content
         """
-        from litellm.llms.anthropic.messages.native.utils import (
-            openai_chat_refusal_text,
-        )
+        from litellm.llms.anthropic.messages.native.utils import refusal_reply_text
 
         from .transformation import LiteLLMAnthropicMessagesAdapter
 
+        self._record_anthropic_stop_fields(chunk.choices[0].delta)
         if chunk.choices[0].finish_reason is not None:
             return False
 
-        refusal_text: Final = openai_chat_refusal_text(chunk.choices[0].delta)
+        refusal_text: Final = refusal_reply_text(chunk.choices[0].delta)
         if refusal_text is not None:
             self._refusal_text = self._refusal_text + refusal_text
 

litellm/llms/anthropic/messages/adapters/transformation.py

@@ -150,7 +150,9 @@ from litellm.llms.anthropic.messages.native.mid_conversation_system import (
     is_system_role_message,
 )
 from litellm.llms.anthropic.messages.native.utils import (
-    openai_chat_refusal_text,
+    anthropic_refusal_stop_details,
+    anthropic_stop_reason,
+    refusal_reply_text,
     refusal_stop_details,
 )
 from litellm.types.llms.anthropic import (
@@ -398,6 +400,10 @@ class LiteLLMAnthropicMessagesAdapter:
 
         return None
 
+    def _streamed_tool_call_signature(self, tool_call_id: str, tool_call: object) -> str | None:
+        id_parts: Final = tool_call_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
+        return id_parts[1] if len(id_parts) > 1 else self._extract_signature_from_tool_call(tool_call)
+
     def _extract_signature_from_tool_use_content(self, content: Mapping[str, object]) -> str | None:
         """
         Extract signature from a tool_use content block's provider_specific_fields.
@@ -1467,7 +1473,7 @@ class LiteLLMAnthropicMessagesAdapter:
                 new_content.append(
                     AnthropicResponseContentBlockText(type="text", text=choice.message.content).model_dump()
                 )
-            if (refusal_text := openai_chat_refusal_text(choice.message)) is not None:
+            if (refusal_text := refusal_reply_text(choice.message)) is not None:
                 new_content.append(AnthropicResponseContentBlockText(type="text", text=refusal_text).model_dump())
             # Handle tool calls (in parallel to text content)
             if choice.message.tool_calls is not None and len(choice.message.tool_calls) > 0:
@@ -1634,7 +1640,15 @@ class LiteLLMAnthropicMessagesAdapter:
             tool_name_mapping=tool_name_mapping,
         )
         refusal_text: Final = next(
-            (text for choice in response.choices if (text := openai_chat_refusal_text(choice.message)) is not None),
+            (text for choice in response.choices if (text := refusal_reply_text(choice.message)) is not None),
+            None,
+        )
+        claude_refusal_stop_details: Final = next(
+            (
+                details
+                for choice in response.choices
+                if (details := anthropic_refusal_stop_details(choice.message)) is not None
+            ),
             None,
         )
 
@@ -1643,8 +1657,10 @@ class LiteLLMAnthropicMessagesAdapter:
 
         ## extract finish reason
         openai_finish_reason: Final = response.choices[0].finish_reason if response.choices else "stop"
-        translated_finish_reason: Final = self._translate_openai_finish_reason_to_anthropic(
-            openai_finish_reason=openai_finish_reason
+        translated_finish_reason: Final = (
+            "pause_turn"
+            if response.choices and anthropic_stop_reason(response.choices[0].message) == "pause_turn"
+            else self._translate_openai_finish_reason_to_anthropic(openai_finish_reason=openai_finish_reason)
         )
         anthropic_finish_reason: Final = (
             "compaction"
@@ -1686,7 +1702,11 @@ class LiteLLMAnthropicMessagesAdapter:
             usage=anthropic_usage,
             content=anthropic_content,
             stop_reason=anthropic_finish_reason,
-            stop_details=(refusal_stop_details(refusal_text) if anthropic_finish_reason == "refusal" else None),
+            stop_details=(
+                (claude_refusal_stop_details or refusal_stop_details(refusal_text))
+                if anthropic_finish_reason == "refusal"
+                else None
+            ),
         )
 
         applied_edits: Final = polyfill_result.applied_edits_for_response() if polyfill_result else None
@@ -1712,10 +1732,7 @@ class LiteLLMAnthropicMessagesAdapter:
             ):
                 raw_id = choice.delta.tool_calls[0].id or str(uuid.uuid4())
                 tool_name = choice.delta.tool_calls[0].function.name or ""
-                thought_sig: str | None = None
-                if THOUGHT_SIGNATURE_SEPARATOR in raw_id:
-                    parts = raw_id.split(THOUGHT_SIGNATURE_SEPARATOR, 1)
-                    thought_sig = parts[1] if len(parts) > 1 else None
+                thought_sig = self._streamed_tool_call_signature(raw_id, choice.delta.tool_calls[0])
                 tool_block: dict[str, object] = {
                     "type": "tool_use",
                     "id": normalize_anthropic_tool_use_id(raw_id),
@@ -1727,7 +1744,7 @@ class LiteLLMAnthropicMessagesAdapter:
                         "signature": thought_sig,
                     }
                 return "tool_use", cast("ContentBlockContentBlockDict", tool_block)
-            elif (choice.delta.content is not None and len(choice.delta.content) > 0) or openai_chat_refusal_text(
+            elif (choice.delta.content is not None and len(choice.delta.content) > 0) or refusal_reply_text(
                 choice.delta
             ) is not None:
                 return "text", TextBlock(type="text", text="")
@@ -1810,7 +1827,7 @@ class LiteLLMAnthropicMessagesAdapter:
             return "thinking_delta", ContentThinkingBlockDelta(type="thinking_delta", thinking=reasoning_content)
         else:
             refusal_text: Final = "".join(
-                refusal for choice in choices if (refusal := openai_chat_refusal_text(choice.delta)) is not None
+                refusal for choice in choices if (refusal := refusal_reply_text(choice.delta)) is not None
             )
             return "text_delta", ContentTextBlockDelta(type="text_delta", text=text + refusal_text)
 

litellm/llms/anthropic/messages/context_management/editors/clear_tool_uses.py

@@ -74,6 +74,7 @@ class _ClearToolUsesFields(BaseModel):
     clear_tool_inputs: bool | tuple[str, ...] | None = None
 
 
+_BLOCK: Final = TypeAdapter(Mapping[str, object])
 _BLOCKS: Final = TypeAdapter(tuple[Mapping[str, object], ...])
 # api.anthropic.com, measured 2026-10-19 and 2026-10-28: its wording for the shapes pydantic words differently
 _API_MESSAGES: Final = MappingProxyType(
@@ -317,12 +318,10 @@ def _clear_tool_results(
         new_blocks: list[object] = []
         mutated = False
         for block in content:
-            if (
-                isinstance(block, dict)
-                and block.get("type") == "tool_result"
-                and block.get("tool_use_id") in ids_to_clear
-            ):
-                new_blocks.append(_cleared_result(block))
+            if (parsed := _parse(_BLOCK, block)) is not None and _block_id(
+                parsed, "tool_result", "tool_use_id"
+            ) in ids_to_clear:
+                new_blocks.append(_cleared_result(parsed))
                 mutated = True
                 cleared += 1
             else:

litellm/llms/anthropic/messages/context_management/inference_view.py

@@ -0,0 +1,138 @@
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
+from typing import Final
+
+from pydantic import JsonValue, TypeAdapter, ValidationError
+from typing_extensions import assert_never
+
+from litellm.types.llms.anthropic import AnthropicMessagesTextParam
+from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat
+
+
+@dataclass(frozen=True, slots=True)
+class InferenceRequestView:
+    messages: Sequence[Mapping[str, object]]
+    system: object | None
+
+
+@dataclass(frozen=True, slots=True)
+class InferenceRequestViewError:
+    error_message: str
+    status_code: int
+
+
+@dataclass(frozen=True, slots=True)
+class CountContextEdits:
+    spec: JsonValue
+    model: str
+    tools: tuple[Mapping[str, object], ...] | None
+    drop_invalid_edits: bool = False
+
+
+_CONTEXT_MANAGEMENT_SPEC: Final[TypeAdapter[Mapping[str, object] | tuple[Mapping[str, object], ...]]] = TypeAdapter(
+    Mapping[str, object] | tuple[Mapping[str, object], ...]
+)
+_SYSTEM_BLOCKS: Final = TypeAdapter(tuple[AnthropicMessagesTextParam, ...])
+
+
+def count_context_edits(
+    context_management: CountContextManagement | None,
+    tools: Sequence[Mapping[str, object]] | None,
+    litellm_params: Mapping[str, object],
+    fallback_model: str,
+) -> CountContextEdits:
+    model: Final = litellm_params.get("model")
+    return CountContextEdits(
+        spec=None if context_management is None else context_management.spec,
+        model=model if isinstance(model, str) else fallback_model,
+        tools=None if tools is None else tuple(tools),
+        drop_invalid_edits=context_management is not None and context_management.drop_invalid_edits,
+    )
+
+
+def _typed_spec(spec: JsonValue) -> Mapping[str, object] | tuple[Mapping[str, object], ...] | None:
+    try:
+        return _CONTEXT_MANAGEMENT_SPEC.validate_python(spec)
+    except ValidationError:
+        return None
+
+
+def _typed_system(system: object | None) -> str | tuple[AnthropicMessagesTextParam, ...] | None | ValidationError:
+    if system is None or isinstance(system, str):
+        return system
+    try:
+        return _SYSTEM_BLOCKS.validate_python(system)
+    except ValidationError as error:
+        return error
+
+
+def _view_without_edits(
+    messages: Sequence[Mapping[str, object]],
+    system: object | None,
+    typed_system: str | Sequence[Mapping[str, object]] | None,
+) -> InferenceRequestView:
+    from litellm.llms.anthropic.messages.context_management.editors.compact import (
+        apply_client_compaction_block_history,
+    )
+
+    history: Final = apply_client_compaction_block_history(
+        messages=[dict(message) for message in messages],
+        system=typed_system
+        if typed_system is None or isinstance(typed_system, str)
+        else [dict(block) for block in typed_system],
+    )
+    if history is None:
+        return InferenceRequestView(messages=messages, system=system)
+    return InferenceRequestView(messages=history.messages, system=history.system)
+
+
+async def _anthropic_inference_view(
+    messages: Sequence[Mapping[str, object]], system: object | None, edits: CountContextEdits
+) -> InferenceRequestView | InferenceRequestViewError:
+    from litellm.llms.anthropic.messages.adapters.handler import prepare_context_managed_request
+    from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
+    from litellm.llms.anthropic.messages.native.utils import normalize_native_messages
+
+    typed_system: Final = _typed_system(system)
+    if isinstance(typed_system, ValidationError):
+        return InferenceRequestView(messages=messages, system=system)
+    normalized: Final = normalize_native_messages(messages)
+    spec: Final = _typed_spec(edits.spec)
+    try:
+        prepared: Final = await prepare_context_managed_request(
+            model=edits.model,
+            messages=normalized,
+            tools=None if edits.tools is None else [dict(tool) for tool in edits.tools],
+            system=[dict(block) for block in typed_system] if isinstance(typed_system, tuple) else typed_system,
+            context_management_spec=(
+                None if spec is None else dict(spec) if isinstance(spec, Mapping) else [dict(edit) for edit in spec]
+            ),
+            litellm_metadata=None,
+            additional_drop_params=None,
+            llm_router=None,
+            count_only=True,
+            drop_invalid_edits=edits.drop_invalid_edits,
+        )
+    except AnthropicContextManagementError as error:
+        if error.status_code >= 500:
+            return _view_without_edits(normalized, system, typed_system)
+        return InferenceRequestViewError(error_message=error.message, status_code=error.status_code)
+    if prepared is None:
+        return InferenceRequestView(messages=normalized, system=system)
+    return InferenceRequestView(messages=prepared.messages, system=prepared.system)
+
+
+async def inference_request_view(
+    messages: Sequence[Mapping[str, object]],
+    system: object | None,
+    messages_format: TokenCountMessagesFormat,
+    edits: CountContextEdits,
+) -> InferenceRequestView | InferenceRequestViewError:
+    """Apply the history rewrites and context edits /v1/messages runs before it converts a body for another provider."""
+    match messages_format:
+        case "openai":
+            return InferenceRequestView(messages=messages, system=system)
+        case "anthropic":
+            return await _anthropic_inference_view(messages, system, edits)
+        case _:
+            assert_never(messages_format)

litellm/llms/anthropic/messages/native/handler.py

@@ -8,6 +8,7 @@
 import asyncio
 import contextvars
 from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Iterator, Mapping, Sequence
+from dataclasses import dataclass
 from functools import partial
 from typing import Final, cast
 
@@ -15,6 +16,7 @@ from pydantic import TypeAdapter
 
 import litellm
 from litellm.anthropic_beta_headers_manager import aload_beta_headers_config
+from litellm.constants import PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO
 from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.exception_mapping_utils import exception_type
@@ -44,7 +46,7 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
 )
 from litellm.types.router import GenericLiteLLMParams
-from litellm.types.utils import CallTypes, LlmProviders
+from litellm.types.utils import CallTypes, LlmProviders, LlmProvidersSet
 from litellm.utils import ProviderConfigManager, client
 
 from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler, request_safeguards_evaluator
@@ -94,6 +96,68 @@ def _responses_mode_is_lost_by_prefix_strip(
     ) and not _bridges_to_responses_api(resolved_model, custom_llm_provider, deployment_model_info)
 
 
+@dataclass(frozen=True, slots=True)
+class _BridgeTarget:
+    model: str
+    api_key: str | None
+    api_base: str | None
+
+
+def _resolve_requested_provider(
+    model: str, custom_llm_provider: str | None, litellm_params: GenericLiteLLMParams
+) -> tuple[str, str, str | None, str | None]:
+    resolved: Final = litellm.get_llm_provider(
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        api_base=litellm_params.api_base,
+        api_key=litellm_params.api_key,
+        litellm_params=litellm_params,
+    )
+    requested_provider: Final = model.partition("/")[0]
+    if (
+        custom_llm_provider in (None, requested_provider)
+        or requested_provider not in LlmProvidersSet
+        or requested_provider in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO
+    ):
+        return resolved
+    own: Final = _resolve_by_model_prefix(model, litellm_params)
+    return own if own is not None and own[1] == custom_llm_provider else resolved
+
+
+def _resolve_by_model_prefix(
+    model: str, litellm_params: GenericLiteLLMParams
+) -> tuple[str, str, str | None, str | None] | None:
+    try:
+        return litellm.get_llm_provider(
+            model=model,
+            api_base=litellm_params.api_base,
+            api_key=litellm_params.api_key,
+            litellm_params=litellm_params.model_copy(update={"custom_llm_provider": None}),
+        )
+    except litellm.BadRequestError:
+        return None
+
+
+def _sets_own_entra_credential(litellm_params: GenericLiteLLMParams) -> bool:
+    return bool(
+        litellm_params.azure_ad_token
+        or (litellm_params.model_extra or {}).get("azure_ad_token_provider") is not None
+        or (litellm_params.tenant_id and litellm_params.client_id and litellm_params.client_secret)
+        or (litellm_params.azure_username and litellm_params.azure_password and litellm_params.client_id)
+    )
+
+
+def _bridge_target(requested: _BridgeTarget, resolved: _BridgeTarget, custom_llm_provider: str) -> _BridgeTarget:
+    requested_provider: Final = requested.model.partition("/")[0]
+    if requested_provider == custom_llm_provider or requested_provider not in LlmProvidersSet:
+        return requested
+    return _BridgeTarget(
+        model=f"{custom_llm_provider}/{resolved.model}",
+        api_key=requested.api_key if requested.api_base else requested.api_key or resolved.api_key,
+        api_base=resolved.api_base,
+    )
+
+
 def _should_route_to_responses_api(
     custom_llm_provider: str | None,
     requested_model: str | None = None,
@@ -624,13 +688,7 @@ def anthropic_messages_handler(
         custom_llm_provider,
         dynamic_api_key,
         dynamic_api_base,
-    ) = litellm.get_llm_provider(
-        model=model,
-        custom_llm_provider=custom_llm_provider,
-        api_base=litellm_params.api_base,
-        api_key=litellm_params.api_key,
-        litellm_params=litellm_params,
-    )
+    ) = _resolve_requested_provider(model, custom_llm_provider, litellm_params)
 
     # Store agentic loop params in logging object for agentic hooks
     # This provides original request context needed for follow-up calls
@@ -656,6 +714,16 @@ def anthropic_messages_handler(
             mock_response=litellm_params.mock_response,
         )
 
+    bridge: Final = _bridge_target(
+        _BridgeTarget(model=original_model, api_key=api_key, api_base=api_base),
+        _BridgeTarget(
+            model=model,
+            api_key=None if _sets_own_entra_credential(litellm_params) else dynamic_api_key,
+            api_base=dynamic_api_base,
+        ),
+        custom_llm_provider,
+    )
+
     # Expand litellm_proxy MCP references through the MCP gateway before dispatch, so every
     # downstream path (native passthrough and both bridges) gets real tools rather than a
     # reference the provider cannot resolve. Popped from kwargs so it never reaches the provider.
@@ -682,8 +750,8 @@ def anthropic_messages_handler(
             top_k=top_k,
             top_p=top_p,
             container=container,
-            api_key=api_key,
-            api_base=api_base,
+            api_key=bridge.api_key,
+            api_base=bridge.api_base,
             client=client,
             custom_llm_provider=custom_llm_provider,
             **kwargs,
@@ -700,7 +768,7 @@ def anthropic_messages_handler(
             return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler(
                 max_tokens=max_tokens,
                 messages=messages,
-                model=original_model,
+                model=bridge.model,
                 metadata=metadata,
                 stop_sequences=stop_sequences,
                 stream=stream,
@@ -716,8 +784,8 @@ def anthropic_messages_handler(
                 top_k=top_k,
                 top_p=top_p,
                 _is_async=is_async,
-                api_key=api_key,
-                api_base=api_base,
+                api_key=bridge.api_key,
+                api_base=bridge.api_base,
                 client=client,
                 custom_llm_provider=custom_llm_provider,
                 **kwargs,
@@ -730,7 +798,7 @@ def anthropic_messages_handler(
         return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler(
             max_tokens=max_tokens,
             messages=messages,
-            model=original_model,
+            model=bridge.model,
             metadata=metadata,
             stop_sequences=stop_sequences,
             stream=stream,
@@ -744,8 +812,8 @@ def anthropic_messages_handler(
             top_k=top_k,
             top_p=top_p,
             _is_async=is_async,
-            api_key=api_key,
-            api_base=api_base,
+            api_key=bridge.api_key,
+            api_base=bridge.api_base,
             client=client,
             custom_llm_provider=custom_llm_provider,
             **kwargs,

litellm/llms/anthropic/messages/native/utils.py

@@ -2,7 +2,7 @@ from collections.abc import Iterable, Mapping, Sequence
 from functools import lru_cache
 from typing import TYPE_CHECKING, Final, cast, get_type_hints
 
-from pydantic import JsonValue, TypeAdapter, ValidationError
+from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError, with_config
 
 from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
 from litellm.litellm_core_utils.core_helpers import as_str_mapping
@@ -162,6 +162,34 @@ def openai_chat_refusal_text(message_or_delta: object) -> str | None:
     return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "refusal")
 
 
+def anthropic_stop_reason(message_or_delta: object) -> str | None:
+    return _mapping_str_field(getattr(message_or_delta, "provider_specific_fields", None), "stop_reason")
+
+
+def refusal_reply_text(message_or_delta: object) -> str | None:
+    if anthropic_stop_reason(message_or_delta) == "refusal":
+        return None
+    return openai_chat_refusal_text(message_or_delta)
+
+
+@with_config(ConfigDict(extra="allow"))
+class _AnthropicRefusalStopDetails(AnthropicStopDetails, total=False):
+    pass
+
+
+_ANTHROPIC_REFUSAL_STOP_DETAILS: Final = TypeAdapter(_AnthropicRefusalStopDetails)
+
+
+def anthropic_refusal_stop_details(message_or_delta: object) -> AnthropicStopDetails | None:
+    provider_specific_fields: Final = getattr(message_or_delta, "provider_specific_fields", None)
+    if _mapping_str_field(provider_specific_fields, "stop_reason") != "refusal":
+        return None
+    try:
+        return _ANTHROPIC_REFUSAL_STOP_DETAILS.validate_python(_mapping_field(provider_specific_fields, "stop_details"))
+    except ValidationError:
+        return refusal_stop_details(openai_chat_refusal_text(message_or_delta))
+
+
 def _responses_message_refusal_text(item: object) -> str | None:
     from openai.types.responses import ResponseOutputMessage, ResponseOutputRefusal
 

litellm/llms/anthropic/messages/responses_adapters/handler.py

@@ -21,7 +21,7 @@ from litellm.litellm_core_utils.asyncify import (
 from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     cache_control_as_prompt_cache_breakpoint,
-    prompt_cache_breakpoint_for_wire,
+    prompt_cache_breakpoint_with_readable_mode,
     with_prompt_cache_breakpoint,
 )
 from litellm.responses.utils import auto_reasoning_summary
@@ -207,7 +207,7 @@ def _without_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str
 def _cache_control_as_wire_safe_prompt_cache_breakpoint(block: Mapping[str, object]) -> Mapping[str, object]:
     if "prompt_cache_breakpoint" not in block:
         return cache_control_as_prompt_cache_breakpoint(block)
-    wire_marker: Final = prompt_cache_breakpoint_for_wire(block["prompt_cache_breakpoint"], drop_params=True)
+    wire_marker: Final = prompt_cache_breakpoint_with_readable_mode(block["prompt_cache_breakpoint"], drop_params=True)
     return cache_control_as_prompt_cache_breakpoint(
         with_prompt_cache_breakpoint(_without_prompt_cache_breakpoint(block), wire_marker)
     )

litellm/llms/anthropic/messages/stream_assembly.py

@@ -155,6 +155,8 @@ def collapse_pure_text_chunks(
             continue
 
         json_str = ev[idx + len("data:") :].strip()
+        if not json_str.startswith("{"):
+            continue
         try:
             data = json.loads(json_str)
         except (json.JSONDecodeError, ValueError):
@@ -206,6 +208,11 @@ def collapse_pure_text_chunks(
     return out
 
 
+def _has_non_object_data(event_str: str) -> bool:
+    data_start: Final = event_str.find("data:")
+    return data_start != -1 and not event_str[data_start + len("data:") :].lstrip().startswith("{")
+
+
 def build_complete_streaming_response_legacy(
     all_chunks: Sequence[str | bytes],
     litellm_logging_obj: LiteLLMLoggingObj,
@@ -243,6 +250,8 @@ def build_complete_streaming_response_legacy(
                 # text payload happens to contain "[DONE]" is not dropped.
                 if any(line.strip() == "data: [DONE]" for line in event_str.split("\n")):
                     continue
+                if _has_non_object_data(event_str):
+                    continue
                 transformed_openai_chunk = anthropic_model_response_iterator.convert_str_chunk_to_generic_chunk(
                     chunk=event_str
                 )
@@ -268,21 +277,25 @@ def build_complete_streaming_response_legacy(
     return complete_streaming_response
 
 
-def extract_sse_data(event_str: str) -> dict | None:
-    """Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
+def _sse_data_payload(event_str: str) -> str | None:
     for line in split_sse_lines(event_str):
         stripped = line.strip()
         if stripped.startswith("data:"):
-            payload = stripped[len("data:") :].strip()
-            if not payload or payload == "[DONE]":
-                return None
-            try:
-                return json.loads(payload)
-            except (ValueError, TypeError):
-                return None
+            return stripped[len("data:") :].strip()
     return None
 
 
+def extract_sse_data(event_str: str) -> dict | None:
+    """Parse the JSON object from the ``data:`` line of an Anthropic SSE event."""
+    payload: Final = _sse_data_payload(event_str)
+    if payload is None or not payload.startswith("{"):
+        return None
+    try:
+        return json.loads(payload)
+    except (ValueError, TypeError):
+        return None
+
+
 def build_usage_only_response_from_chunks(
     all_chunks: Sequence[str | bytes],
     model: str,
@@ -415,27 +428,81 @@ _ANTHROPIC_EVENT_TYPES: Final = frozenset(
 )
 
 
+_ERROR_PAYLOAD_KEYS: Final = frozenset({"type", "error", "request_id"})
+
+
+def _is_anthropic_event_type(event_type: object) -> bool:
+    return isinstance(event_type, str) and event_type in _ANTHROPIC_EVENT_TYPES
+
+
 def is_raw_sse_stream(all_chunks: Sequence[object]) -> bool:
     return any(isinstance(chunk, (str, bytes)) for chunk in all_chunks)
 
 
-def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
-    raw: Final = b"".join(
+def _joined_sse_bytes(all_chunks: Sequence[object]) -> bytes:
+    return b"".join(
         chunk if isinstance(chunk, bytes) else chunk.encode("utf-8")
         for chunk in all_chunks
         if isinstance(chunk, (str, bytes))
     )
+
+
+def _joined_sse_stream(all_chunks: Sequence[object]) -> str | None:
     try:
-        return codecs.getincrementaldecoder("utf-8")().decode(raw, final=False)
+        return codecs.getincrementaldecoder("utf-8")().decode(_joined_sse_bytes(all_chunks), final=False)
     except UnicodeDecodeError:
         return None
 
 
+def _sse_event_data(event: str) -> Mapping[str, object] | None:
+    return as_str_mapping(extract_sse_data(event))
+
+
+_JSON_ESCAPED_LINE_BREAKS: Final = MappingProxyType(
+    str.maketrans({"\x85": "\\u0085", "\u2028": "\\u2028", "\u2029": "\\u2029"})
+)
+
+
+def _sse_frames(sse_stream: str) -> Sequence[str]:
+    return "\n".join(split_sse_lines(sse_stream)).split("\n\n")
+
+
+def _frames_of_the_widest_reader(sse_stream: str) -> Sequence[str]:
+    """The stream's frames as httpx's ``str.splitlines`` reads them, past a form feed or U+2028 in any line."""
+    return "\n".join(sse_stream.splitlines()).split("\n\n")
+
+
+def _sse_field(line: str) -> tuple[str, str]:
+    name, _, value = line.partition(":")
+    return name, value.removeprefix(" ")
+
+
+def _sse_fields(frame: str) -> tuple[tuple[str, str], ...]:
+    """The frame's fields as a client reads them: a name matches exactly and a comment is no field."""
+    return tuple(_sse_field(line) for line in frame.split("\n") if line and not line.startswith(":"))
+
+
+def _carries_no_data(fields: Sequence[tuple[str, str]]) -> bool:
+    match fields:
+        case () | (("event", _),) | (("data", "[DONE]"),):
+            return True
+        case _:
+            return False
+
+
+def _json_mapping(payload: str) -> Mapping[str, object] | None:
+    try:
+        value: Final[object] = json.loads(payload)
+    except ValueError:
+        return None
+    return as_str_mapping(value)
+
+
 def _parsed_sse_events(sse_stream: str) -> tuple[Mapping[str, object], ...]:
     return tuple(
         event_data
         for event in split_sse_chunk_into_events(sse_stream)
-        if (event_data := extract_sse_data(event)) is not None
+        if (event_data := _sse_event_data(event)) is not None
     )
 
 
@@ -444,7 +511,8 @@ def _anthropic_message_start(sse_stream: str) -> Mapping[str, object] | None:
         (
             message
             for event_data in _parsed_sse_events(sse_stream)
-            if event_data.get("type") == "message_start" and isinstance(message := event_data.get("message"), dict)
+            if event_data.get("type") == "message_start"
+            and (message := as_str_mapping(event_data.get("message"))) is not None
         ),
         None,
     )
@@ -482,11 +550,151 @@ def is_anthropic_sse_stream(all_chunks: Sequence[object]) -> bool:
     sse_stream: Final = _joined_sse_stream(all_chunks)
     if sse_stream is None:
         return False
-    return any(event.get("type") in _ANTHROPIC_EVENT_TYPES for event in _parsed_sse_events(sse_stream))
+    return any(_is_anthropic_event_type(event.get("type")) for event in _parsed_sse_events(sse_stream))
+
+
+def _is_json_no_reader_can_take(payload: str) -> bool:
+    try:
+        value: Final[object] = json.loads(payload)
+    except ValueError:
+        return False
+    if value is None:
+        return False
+    event: Final = as_str_mapping(value)
+    return event is None or isinstance(event.get("type"), (list, dict))
+
+
+def has_unreadable_json_event(all_chunks: Sequence[object]) -> bool:
+    """Whether a frame's data is JSON that no reader of these streams can take as an event."""
+    sse_stream: Final = codecs.getincrementaldecoder("utf-8")(errors="replace").decode(
+        _joined_sse_bytes(all_chunks), final=False
+    )
+    return any(_is_json_no_reader_can_take(_sse_data_payload(event) or "") for event in _sse_frames(sse_stream))
+
+
+def anthropic_sse_event_type(event: str) -> object:
+    return next((event_data.get("type") for event_data in _parsed_sse_events(event)), None)
+
+
+def anthropic_sse_event_types(all_chunks: Sequence[object]) -> frozenset[str]:
+    """The string ``type`` of every event the stream sent."""
+    event_types: Final = (event.get("type") for event in _parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
+    return frozenset(event_type for event_type in event_types if isinstance(event_type, str))
+
+
+def _is_named_error_event(event: str) -> bool:
+    names: Final = tuple(value for name, value in _sse_fields(event) if name == "event")
+    return names[-1:] == ("error",)
+
+
+def _is_error_event(event: str) -> bool:
+    event_type: Final = anthropic_sse_event_type(event)
+    return event_type == "error" or (_is_named_error_event(event) and not _is_anthropic_event_type(event_type))
+
+
+def _frames_a_client_dispatches(sse_stream: str) -> Sequence[str]:
+    return "\n".join((*split_sse_lines(sse_stream), "")).split("\n\n")[:-1]
+
+
+def _error_events(sse_stream: str) -> tuple[str, ...]:
+    return tuple(event for event in _frames_a_client_dispatches(sse_stream) if _is_error_event(event))
+
+
+def _plain_error_event(event: str) -> str:
+    payload: Final = _sse_data_payload(event) or ""
+    one_line: Final = payload.translate(_JSON_ESCAPED_LINE_BREAKS) if _json_mapping(payload) is not None else payload
+    return "\n".join(("event: error", *(f"data: {line}" for line in one_line.splitlines()[:1])))
+
+
+def anthropic_sse_error_events(all_chunks: Sequence[object]) -> tuple[str, ...]:
+    """The ``error`` events of the stream, each as an ``event:`` line and the ``data:`` line assembly read.
+
+    A stream whose upstream fails after it started ends with an ``error`` event in place of
+    ``message_stop``. That frame is the outcome the client is owed, not content, so assembly through
+    the upstream error leaves it out and a caller re-emitting a rewritten response puts it back,
+    without any other line of the frame, since that is content no guardrail read.
+    A frame counts by its data ``type``, or by its SSE ``event:`` name unless its data is another
+    Anthropic event, since a client SDK raises on the name alone and an upstream that is not
+    Anthropic may send anything as the data.
+    """
+    return tuple(_plain_error_event(event) for event in _error_events(_joined_sse_stream(all_chunks) or ""))
+
+
+def anthropic_sse_stopped_block_index(event: str) -> object:
+    return next(
+        (
+            event_data.get("index")
+            for event_data in _parsed_sse_events(event)
+            if event_data.get("type") == "content_block_stop"
+        ),
+        None,
+    )
+
+
+def _block_index(event_data: Mapping[str, object]) -> int | None:
+    index: Final = event_data.get("index")
+    return index if isinstance(index, int) else None
+
+
+def _started_blocks(events: Sequence[Mapping[str, object]]) -> Mapping[int, Mapping[str, object]]:
+    starts: Final = (event for event in events if event.get("type") == "content_block_start")
+    return MappingProxyType(
+        {
+            index: block
+            for event in starts
+            if (index := _block_index(event)) is not None
+            and (block := as_str_mapping(event.get("content_block"))) is not None
+        }
+    )
+
+
+def anthropic_sse_started_blocks(all_chunks: Sequence[object]) -> Mapping[int, Mapping[str, object]]:
+    """The ``content_block`` each ``content_block_start`` of the stream opened, by block index."""
+    return _started_blocks(_parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
+
+
+def _unfinished_blocks(events: Sequence[Mapping[str, object]]) -> Mapping[int, Mapping[str, object]]:
+    stopped: Final = frozenset(_block_index(event) for event in events if event.get("type") == "content_block_stop")
+    return MappingProxyType({index: block for index, block in _started_blocks(events).items() if index not in stopped})
+
+
+def anthropic_sse_unfinished_blocks(all_chunks: Sequence[object]) -> Mapping[int, Mapping[str, object]]:
+    """The blocks the stream opened and never stopped, which only a failed upstream leaves."""
+    return _unfinished_blocks(_parsed_sse_events(_joined_sse_stream(all_chunks) or ""))
+
+
+def _sse_events_with_data(sse_stream: str) -> tuple[tuple[str, Mapping[str, object]], ...]:
+    return tuple(
+        (event, event_data)
+        for event in split_sse_chunk_into_events(sse_stream)
+        if (event_data := _sse_event_data(event)) is not None
+    )
+
+
+def _content_events_before_upstream_error(sse_stream: str) -> tuple[str, ...]:
+    """The stream's events minus its upstream ``error`` and every cut-off block assembly cannot read.
+
+    A cut-off text block is still text the client was sent, so it stays. A cut-off tool call's
+    input is a JSON prefix, and a cut-off thinking block has no signature, so neither is a block the
+    client can act on, and both are left out.
+    """
+    if not _error_events(sse_stream):
+        return (sse_stream,)
+    events: Final = _sse_events_with_data(sse_stream)
+    cut_off: Final = frozenset(
+        index
+        for index, block in _unfinished_blocks(tuple(event_data for _, event_data in events)).items()
+        if block.get("type") != "text"
+    )
+    return tuple(
+        event
+        for event, event_data in events
+        if event_data.get("type") != "error" and _block_index(event_data) not in cut_off
+    )
 
 
 def assemble_anthropic_sse_stream(
-    all_chunks: Sequence[object], *, restore_identity: bool = False
+    all_chunks: Sequence[object], *, restore_identity: bool = False, through_upstream_error: bool = False
 ) -> ModelResponse | None:
     """Assemble raw Anthropic SSE frames into a ModelResponse.
 
@@ -494,6 +702,11 @@ def assemble_anthropic_sse_stream(
     assembler does not carry through. It is off by default so callers that re-emit the assembled
     response keep the wire shape they had before this helper was shared. The writes land on a
     freshly built object that is unreachable from caller state until returned.
+
+    A stream whose upstream failed after it started ends with an ``error`` event in place of
+    ``message_stop`` and assembles to None, unless ``through_upstream_error`` asks for the content
+    the client was sent before the failure. That is for a guardrail that scans the stream and then
+    passes the upstream error on; a caller recording what was served keeps None.
     """
     sse_stream: Final = _joined_sse_stream(all_chunks)
     if sse_stream is None:
@@ -502,9 +715,12 @@ def assemble_anthropic_sse_stream(
     if message_start is None:
         return None
     model: Final = message_start.get("model") if restore_identity else None
+    content_events: Final = (
+        _content_events_before_upstream_error(sse_stream) if through_upstream_error else (sse_stream,)
+    )
     try:
         assembled: Final = build_complete_streaming_response(
-            all_chunks=(sse_stream,),
+            all_chunks=content_events,
             litellm_logging_obj=None,  # pyright: ignore[reportArgumentType]  # only forwarded to stream_chunk_builder, which accepts None
             model=model if isinstance(model, str) else "",
         )
@@ -522,6 +738,21 @@ def assemble_anthropic_sse_stream(
     return assembled
 
 
+def _is_error_payload(payload: str) -> bool:
+    event: Final = _json_mapping(payload)
+    if event is None or not _ERROR_PAYLOAD_KEYS.issuperset(event):
+        return False
+    return event.get("type") == "error" or ("type" not in event and isinstance(event.get("error"), Mapping))
+
+
+def _is_error_frame(fields: Sequence[tuple[str, str]]) -> bool:
+    match fields:
+        case (("data", payload),) | (("event", "error"), ("data", payload)):
+            return _is_error_payload(payload)
+        case _:
+            return False
+
+
 def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
     """Whether the buffered stream carries nothing but error frames.
 
@@ -537,7 +768,6 @@ def is_sse_error_stream(all_chunks: Sequence[object]) -> bool:
     sse_stream: Final = _joined_sse_stream(all_chunks)
     if sse_stream is None:
         return False
-    events: Final = _parsed_sse_events(sse_stream)
-    return len(events) > 0 and all(
-        event.get("type") == "error" or isinstance(event.get("error"), Mapping) for event in events
-    )
+    frames: Final = tuple(_sse_fields(frame) for frame in _frames_of_the_widest_reader(sse_stream))
+    with_data: Final = tuple(fields for fields in frames if not _carries_no_data(fields))
+    return len(with_data) > 0 and all(_is_error_frame(fields) for fields in with_data)

litellm/llms/azure/chat/gpt_reasoning_transformation.py

@@ -1,6 +1,6 @@
 """Support for Azure OpenAI gpt-5 model family."""
 
-from collections.abc import Collection
+from collections.abc import Collection, Mapping
 from typing import ClassVar, Final
 
 import litellm
@@ -105,7 +105,7 @@ class AzureOpenAIGPT5Config(AzureOpenAIConfig, OpenAIGPT5Config):
         model: str,
         messages: list[AllMessageValues],
         optional_params: dict,
-        litellm_params: dict,
+        litellm_params: Mapping[str, object],
         headers: dict,
     ) -> dict:
         model = model.replace(self.GPT5_SERIES_ROUTE, "")

litellm/llms/azure/chat/gpt_transformation.py

@@ -5,12 +5,16 @@ from typing import TYPE_CHECKING, Any, Final
 from httpx._models import Headers, Response
 
 import litellm
+from litellm.integrations.anthropic_cache_control_hook import deployment_prompt_cache_breakpoint_support
+from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    chat_completion_prompt_cache_breakpoints_for_wire,
     drop_tool_reference_parts_from_tool_messages,
     flatten_combinators_and_drop_non_python_regex_patterns,
     hoist_images_from_tool_messages,
     system_messages_first,
     tool_with_sanitized_parameters,
+    without_prompt_cache_breakpoints,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
     convert_to_azure_openai_messages,
@@ -290,11 +294,16 @@ class AzureOpenAIConfig(BaseConfig):
         model: str,
         messages: list[AllMessageValues],
         optional_params: dict,
-        litellm_params: dict,
+        litellm_params: Mapping[str, object],
         headers: dict,
     ) -> dict:
         ordered_messages: Final = system_messages_first(messages) if litellm.openai_system_messages_first else messages
-        stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(ordered_messages)
+        wire_messages: Final = (
+            self._with_wire_safe_prompt_cache_breakpoints(model, ordered_messages, litellm_params)
+            if drops_unsupported_params(litellm_params.get("drop_params"))
+            else ordered_messages
+        )
+        stripped_messages: Final = drop_tool_reference_parts_from_tool_messages(wire_messages)
         azure_messages: Final = convert_to_azure_openai_messages(hoist_images_from_tool_messages(stripped_messages))
         request_params: Final = MappingProxyType(
             {
@@ -310,6 +319,14 @@ class AzureOpenAIConfig(BaseConfig):
             **sanitized_tools_update(optional_params),
         }
 
+    @staticmethod
+    def _with_wire_safe_prompt_cache_breakpoints(
+        model: str, messages: list[AllMessageValues], litellm_params: Mapping[str, object]
+    ) -> list[AllMessageValues]:
+        if deployment_prompt_cache_breakpoint_support(model, "azure", litellm_params) is False:
+            return [without_prompt_cache_breakpoints(message) for message in messages]
+        return [chat_completion_prompt_cache_breakpoints_for_wire(message, drop_params=True) for message in messages]
+
     def transform_response(
         self,
         model: str,

litellm/llms/azure/responses/transformation.py

@@ -1,6 +1,7 @@
 import re
-from collections.abc import Mapping
+from collections.abc import Callable, Mapping
 from copy import deepcopy
+from functools import partial
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal
 from urllib.parse import urlsplit
@@ -11,6 +12,7 @@ from pydantic import TypeAdapter
 
 from litellm._logging import verbose_logger
 from litellm.constants import AZURE_DEFAULT_RESPONSES_API_VERSION
+from litellm.litellm_core_utils.prompt_templates.common_utils import prompt_cache_breakpoint_within
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.azure.chat.gpt_reasoning_transformation import AzureOpenAIGPT5Config
 from litellm.llms.azure.common_utils import BaseAzureLLM
@@ -29,6 +31,7 @@ else:
     LiteLLMLoggingObj = Any
 
 _DECODED_JSON: Final = TypeAdapter(object)
+_AZURE_RESPONSES_PROMPT_CACHE_BREAKPOINT: Final = TypeAdapter(AzureResponsesPromptCacheBreakpoint)
 
 
 _DATED_GA_API_VERSION: Final = re.compile(r"\d{4}-\d{2}-\d{2}")
@@ -81,6 +84,13 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
     def _is_gpt_5_model(model: str) -> bool:
         return AzureOpenAIGPT5Config.is_model_gpt_5_model(model)
 
+    def _prompt_cache_breakpoint_to_wire(
+        self, litellm_params: GenericLiteLLMParams, reads_breakpoints: bool, known_support: bool | None
+    ) -> Callable[[object], object]:
+        if reads_breakpoints or known_support is True:
+            return partial(prompt_cache_breakpoint_within, _AZURE_RESPONSES_PROMPT_CACHE_BREAKPOINT)
+        return super()._prompt_cache_breakpoint_to_wire(litellm_params, reads_breakpoints, known_support)
+
     @staticmethod
     def _supports_reasoning_effort_none(model: str) -> bool:
         return AzureOpenAIGPT5Config.supports_reasoning_effort_level(model, "none")

litellm/llms/azure_ai/anthropic/count_tokens/handler.py

@@ -12,6 +12,7 @@ from pydantic import JsonValue
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.anthropic_beta_headers_manager import aload_beta_headers_config
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.llms.anthropic.common_utils import AnthropicError
@@ -39,6 +40,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
         tools: list[dict[str, JsonValue]] | None = None,
         system: object = None,
         optional_params: Mapping[str, JsonValue] | None = None,
+        context_management: JsonValue = None,
     ) -> dict[str, object]:
         """
         Handle a CountTokens request using httpx with Azure authentication.
@@ -70,6 +72,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
                 tools=tools,
                 system=system,
                 optional_params=optional_params,
+                context_management=context_management,
                 drop_params=drops_unsupported_params((litellm_params or {}).get("drop_params")),
             )
 
@@ -80,11 +83,14 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
 
             verbose_logger.debug("Making request to: %s", endpoint_url)
 
+            if context_management is not None:
+                await aload_beta_headers_config()
             # Get required headers with Azure authentication
             headers: Final = self.get_required_headers(
                 api_key=api_key,
                 litellm_params=litellm_params,
                 messages=messages,
+                context_management=context_management,
             )
             if not {"x-api-key", "Authorization"} & headers.keys():
                 raise AnthropicError(

litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py

@@ -135,6 +135,7 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
                 tools=tools,
                 system=system,
                 optional_params=count_options,
+                context_management=None if context_management is None else context_management.spec,
             )
 
             if result is not None:

litellm/llms/azure_ai/anthropic/count_tokens/transformation.py

@@ -8,9 +8,16 @@ from collections.abc import Sequence
 from typing import Final
 from urllib.parse import urlsplit
 
+from pydantic import JsonValue
+
+from litellm.anthropic_beta_headers_manager import filter_and_transform_beta_headers
 from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
 from litellm.litellm_core_utils.url_utils import join_api_base
-from litellm.llms.anthropic.common_utils import count_tokens_feature_betas, merge_anthropic_beta_headers
+from litellm.llms.anthropic.common_utils import (
+    context_management_betas,
+    count_tokens_feature_betas,
+    merge_anthropic_beta_headers,
+)
 from litellm.llms.anthropic.count_tokens.transformation import (
     AnthropicCountTokensConfig,
 )
@@ -31,6 +38,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
         api_key: str | None,
         litellm_params: dict[str, object] | None = None,
         messages: Sequence[object] = (),
+        context_management: JsonValue = None,
     ) -> dict[str, str]:
         """
         Get the required headers for the Azure AI Anthropic CountTokens API.
@@ -42,6 +50,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
             api_key: The Azure AI API key
             litellm_params: Optional LiteLLM parameters for additional auth config
             messages: The request messages, read for betas the counted features need
+            context_management: The edit spec the count forwards, read for the betas its edits need
 
         Returns:
             Dictionary of required headers with both x-api-key and Azure authentication
@@ -51,7 +60,11 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
             "Content-Type": "application/json",
             "anthropic-version": "2023-06-01",
             "anthropic-beta": merge_anthropic_beta_headers(
-                ANTHROPIC_TOKEN_COUNTING_BETA_VERSION, count_tokens_feature_betas("azure_ai", {}, messages)
+                ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
+                (
+                    *count_tokens_feature_betas("azure_ai", {}, messages),
+                    *filter_and_transform_beta_headers(list(context_management_betas(context_management)), "azure_ai"),
+                ),
             ),
             **({"x-api-key": api_key} if api_key else {}),
         }

litellm/llms/base_llm/files/batch_records.py

@@ -19,6 +19,8 @@ def responses_batch_body_to_chat_body(
     openai_request_body: Mapping[str, object],
     custom_llm_provider: str | None = None,
     drop_params: bool | None = None,
+    *,
+    caller_input: str | ResponseInputParam | None = None,
 ) -> dict[str, object]:  # mutable-ok: provider transforms take the bridged chat body as a plain dict
     """
     Rewrite the body of an OpenAI `/v1/responses` batch record as a Chat Completions body.
@@ -52,5 +54,6 @@ def responses_batch_body_to_chat_body(
         responses_api_request=responses_request,
         custom_llm_provider=custom_llm_provider,
         drop_params=drop_params,
+        caller_input=caller_input,
         metadata=openai_request_body.get("metadata"),
     )

litellm/llms/bedrock/chat/agentcore/transformation.py

@@ -544,6 +544,17 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
 
         return AgentCoreParsedResponse(content=content, usage=usage_data, final_message=final_message)
 
+    @staticmethod
+    def _message_stop_chunk(model: str) -> ModelResponseStream:
+        chunk: Final = ModelResponseStream(
+            id=f"chatcmpl-{uuid.uuid4()}",
+            created=0,
+            model=model,
+            object="chat.completion.chunk",
+        )
+        chunk.choices = [StreamingChoices(finish_reason="stop", index=0, delta=Delta())]
+        return chunk
+
     def _stream_agentcore_response_sync(
         self,
         response: httpx.Response,
@@ -553,6 +564,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         Internal sync generator that parses SSE and yields ModelResponse chunks.
         """
         buffer = ""
+        sent_finish = False  # rebind-ok: flips when a finish chunk goes out
+        message_stopped = False  # rebind-ok: flips on messageStop, the only end some runtimes send
         for text_chunk in response.iter_text():
             buffer += text_chunk
 
@@ -624,7 +637,9 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
                                     total_tokens=usage_data.get("totalTokens", 0),
                                 ),
                             )
+                            sent_finish = True
                             yield chunk
+                        message_stopped |= "messageStop" in event_payload
 
                     # Process final message
                     if "message" in data_obj and isinstance(data_obj["message"], dict):
@@ -641,12 +656,16 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
                                 delta=Delta(),
                             )
                         ]
+                        sent_finish = True
                         yield chunk
 
                 except json.JSONDecodeError:
                     verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100])
                     continue
 
+        if message_stopped and not sent_finish:
+            yield self._message_stop_chunk(model)
+
     def get_sync_custom_stream_wrapper(
         self,
         model: str,
@@ -775,6 +794,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         Internal async generator that parses SSE and yields ModelResponse chunks.
         """
         buffer = ""
+        sent_finish = False  # rebind-ok: flips when a finish chunk goes out
+        message_stopped = False  # rebind-ok: flips on messageStop, the only end some runtimes send
         async for text_chunk in response.aiter_text():
             buffer += text_chunk
 
@@ -846,7 +867,9 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
                                     total_tokens=usage_data.get("totalTokens", 0),
                                 ),
                             )
+                            sent_finish = True
                             yield chunk
+                        message_stopped |= "messageStop" in event_payload
 
                     # Process final message
                     if "message" in data_obj and isinstance(data_obj["message"], dict):
@@ -863,12 +886,16 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
                                 delta=Delta(),
                             )
                         ]
+                        sent_finish = True
                         yield chunk
 
                 except json.JSONDecodeError:
                     verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100])
                     continue
 
+        if message_stopped and not sent_finish:
+            yield self._message_stop_chunk(model)
+
     async def get_async_custom_stream_wrapper(
         self,
         model: str,

litellm/llms/bedrock/common_utils.py

@@ -43,6 +43,7 @@ from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.bedrock.request_metadata import bedrock_request_metadata_is_owned
 from litellm.secret_managers.main import get_secret, get_secret_str
+from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
 from litellm.types.llms.bedrock import AWS_AUTH_PARAM_KEYS, AwsAuthParams
 
 if TYPE_CHECKING:
@@ -50,6 +51,12 @@ if TYPE_CHECKING:
     from litellm.types.llms.openai import AllMessageValues
 
 BEDROCK_INVOKE_TRACE_FIELD: Final = "amazon-bedrock-trace"
+BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS: Final[Mapping[str, str]] = MappingProxyType(
+    {
+        "compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
+        "clear_tool_uses_20250919": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
+    }
+)
 
 
 _ERROR_REQUEST_URL: Final = "https://docs.litellm.ai/docs"

litellm/llms/bedrock/count_tokens/bedrock_token_counter.py

@@ -13,9 +13,14 @@ from typing_extensions import assert_never
 import litellm
 from litellm._logging import verbose_logger
 from litellm.llms.anthropic.chat.transformation import AnthropicConfig
+from litellm.llms.anthropic.common_utils import context_management_betas, context_management_with_edits
 from litellm.llms.anthropic.count_tokens.transformation import ChatCountTranslationError, anthropic_count_fields
 from litellm.llms.base_llm.base_utils import BaseTokenCounter
-from litellm.llms.bedrock.common_utils import BedrockError, get_bedrock_base_model
+from litellm.llms.bedrock.common_utils import (
+    BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS,
+    BedrockError,
+    get_bedrock_base_model,
+)
 from litellm.llms.bedrock.count_tokens.handler import BedrockCountTokensHandler
 from litellm.llms.bedrock.count_tokens.mantle_handler import BedrockMantleCountTokensHandler
 from litellm.llms.bedrock_mantle.common_utils import is_mantle_claude_model
@@ -74,6 +79,16 @@ def _runtime_rejected_claude_model(outcome: CountTokensOutcome, resolved_model:
     )
 
 
+def _mantle_retry_outcome(
+    runtime: CountTokensOutcome, mantle: CountTokensOutcome, carried_edits: bool
+) -> CountTokensOutcome:
+    match runtime, mantle:
+        case CountTokensFailure(status_code=400), CountTokensFailure(status_code=404) if carried_edits:
+            return runtime
+        case _:
+            return mantle
+
+
 def _request_data(
     model: str, messages: Sequence[Mapping[str, object]], tools: Sequence[Mapping[str, object]] | None, system: object
 ) -> Mapping[str, object]:
@@ -85,6 +100,33 @@ def _request_data(
     }
 
 
+def _invoke_model_message(message: Mapping[str, object]) -> Mapping[str, object]:
+    content: Final = message.get("content")
+    return (
+        {**message, "content": [{"type": "text", "text": content}]} if isinstance(content, str) and content else message
+    )
+
+
+def _invoke_supports_edit(edit: JsonValue) -> bool:
+    edit_type: Final = edit.get("type") if isinstance(edit, dict) else None
+    return isinstance(edit_type, str) and edit_type in BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS
+
+
+def _invoke_context_management(context_management: CountContextManagement | None) -> Mapping[str, JsonValue]:
+    spec: Final = context_management_with_edits(None if context_management is None else context_management.spec)
+    edits: Final = spec.get("edits") if isinstance(spec, dict) else None
+    supported_edits: Final = tuple(filter(_invoke_supports_edit, edits if isinstance(edits, list) else ()))
+    if not isinstance(spec, dict) or not supported_edits:
+        return MappingProxyType({})
+    supported_spec: Final[JsonValue] = {**spec, "edits": list(supported_edits)}
+    return MappingProxyType(
+        {
+            "context_management": supported_spec,
+            "anthropic_beta": list(context_management_betas(supported_spec)),
+        }
+    )
+
+
 class BedrockTokenCounter(BaseTokenCounter):
     """Counts on the configured Bedrock endpoint, with Mantle retry when Runtime rejects a Claude model"""
 
@@ -167,11 +209,19 @@ class BedrockTokenCounter(BaseTokenCounter):
         )
         if isinstance(translated, ChatCountTranslationError):
             return translated.response(request_model, model_to_use, primary_tokenizer)
-        runtime_request: Final = (
-            _request_data(model_to_use, messages, tools, system)
-            if translated is None
-            else _request_data(model_to_use, translated.messages, translated.tools, translated.system)
+        native_context_management: Final = _invoke_context_management(
+            context_management if is_mantle_claude_model(resolved_model) else None
         )
+        counted_messages: Final = messages if translated is None else translated.messages
+        runtime_request: Final = {
+            **_request_data(
+                model_to_use,
+                tuple(map(_invoke_model_message, counted_messages)) if native_context_management else counted_messages,
+                tools if translated is None else translated.tools,
+                system if translated is None else translated.system,
+            ),
+            **native_context_management,
+        }
         mantle_request: Final = runtime_request if translated is None else {**runtime_request, **translated.options}
 
         primary: Final = (
@@ -184,8 +234,12 @@ class BedrockTokenCounter(BaseTokenCounter):
             )
         )
         outcome: Final = (
-            await self._count_with(
-                self._mantle_handler, MANTLE_TOKENIZER_TYPE, mantle_request, litellm_params, resolved_model
+            _mantle_retry_outcome(
+                primary,
+                await self._count_with(
+                    self._mantle_handler, MANTLE_TOKENIZER_TYPE, mantle_request, litellm_params, resolved_model
+                ),
+                carried_edits=bool(native_context_management),
             )
             if not use_mantle and _runtime_rejected_claude_model(primary, resolved_model)
             else primary

litellm/llms/bedrock/count_tokens/mantle_handler.py

@@ -8,7 +8,7 @@ from typing_extensions import NotRequired, ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.url_utils import join_api_base
-from litellm.llms.anthropic.common_utils import tools_without_custom_field
+from litellm.llms.anthropic.common_utils import context_management_betas, tools_without_custom_field
 from litellm.llms.anthropic.count_tokens.transformation import COUNT_TOKEN_OPTION_NAMES, AnthropicCountTokensConfig
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM, run_aws_signing
 from litellm.llms.bedrock.common_utils import BedrockError, build_mantle_messages_url
@@ -22,6 +22,7 @@ class MantleCountTokensRequest(TypedDict):
     messages: ReadOnly[list[dict[str, JsonValue]]]
     system: ReadOnly[NotRequired[JsonValue]]
     tools: ReadOnly[NotRequired[list[dict[str, JsonValue]]]]
+    context_management: ReadOnly[NotRequired[JsonValue]]
 
 
 _COUNT_REQUEST: Final = TypeAdapter(MantleCountTokensRequest)
@@ -74,13 +75,19 @@ class BedrockMantleCountTokensHandler(BaseAWSLLM):
                 optional_params=_COUNT_OPTIONS.validate_python(
                     {name: value for name, value in request_data.items() if name in COUNT_TOKEN_OPTION_NAMES}
                 ),
+                context_management=request.get("context_management"),
             )
+            betas: Final = context_management_betas(request.get("context_management"))
             verbose_logger.debug("Making bedrock-mantle count_tokens request to: %s", endpoint_url)
             api_key: Final = litellm_params.get("api_key")
             signed_headers, signed_body = await run_aws_signing(
                 self._sign_request,
                 service_name="bedrock",
-                headers={"Content-Type": "application/json", "anthropic-version": MANTLE_ANTHROPIC_VERSION},
+                headers={
+                    "Content-Type": "application/json",
+                    "anthropic-version": MANTLE_ANTHROPIC_VERSION,
+                    **({"anthropic-beta": ",".join(betas)} if betas else {}),
+                },
                 optional_params=litellm_params,
                 request_data=body,
                 api_base=endpoint_url,

litellm/llms/bedrock/files/transformation.py

@@ -42,6 +42,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     extract_file_data,
     text_completion_prompt_to_messages,
 )
+from litellm.llms.anthropic.common_utils import AnthropicError
 from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.files.batch_records import responses_batch_body_to_chat_body
@@ -347,7 +348,14 @@ class _BatchInputLine:
     record: _OpenAIBatchRecord
 
 
-_BATCH_RECORD_CONVERSION_ERRORS: Final = (AttributeError, LookupError, TypeError, ValueError, BadRequestError)
+_BATCH_RECORD_CONVERSION_ERRORS: Final = (
+    AttributeError,
+    LookupError,
+    TypeError,
+    ValueError,
+    BadRequestError,
+    AnthropicError,
+)
 
 
 def _batch_line_error(line_number: int, problem: str) -> BedrockError:

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude_transformation.py

@@ -37,6 +37,7 @@ from litellm.llms.bedrock.chat.invoke_transformations.base_invoke_transformation
     bedrock_invoke_guardrail_headers,
 )
 from litellm.llms.bedrock.common_utils import (
+    BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS,
     BedrockError,
     apply_bedrock_invoke_structured_output,
     bedrock_supports_tool_search,
@@ -499,12 +500,7 @@ class AmazonAnthropicClaudeMessagesConfig(
     # Bedrock InvokeModel DOES support ``clear_tool_uses_20250919`` under the
     # ``context-management-2025-06-27`` beta. AWS docs:
     # https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-tool-use.md
-    _BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: Mapping[str, str] = MappingProxyType(
-        {
-            "compact_20260112": ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_01_12.value,
-            "clear_tool_uses_20250919": ANTHROPIC_BETA_HEADER_VALUES.CONTEXT_MANAGEMENT_2025_06_27.value,
-        }
-    )
+    _BEDROCK_INVOKE_SUPPORTED_CONTEXT_MANAGEMENT_EDITS: Mapping[str, str] = BEDROCK_INVOKE_CONTEXT_MANAGEMENT_EDIT_BETAS
 
     @classmethod
     def _filter_context_management_for_bedrock_invoke(

litellm/llms/bedrock_mantle/responses/transformation.py

@@ -15,7 +15,8 @@ role / access key / profile / web identity), signed via the shared
 BaseAWSLLM._sign_request after the request body is finalized.
 """
 
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
+from functools import partial
 from typing import Final, cast  # noqa: TID251  # map_openai_params returns the filtered params as a bare dict
 
 import httpx
@@ -23,6 +24,7 @@ import httpx
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
+from litellm.litellm_core_utils.prompt_templates.common_utils import chat_completion_prompt_cache_breakpoint_for_wire
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.responses.codex_compat import drop_unsupported_tools, normalize_codex_input_items
@@ -62,6 +64,10 @@ _BEDROCK_MANTLE_SUPPORTED_SERVICE_TIERS: Final = frozenset({"auto", "default"})
 _BEDROCK_MANTLE_OPENAI_PATH_SUPPORTED_REASONING_SUMMARIES: Final = frozenset({"auto"})
 
 
+def _prompt_cache_breakpoint_as_sent_or_none(marker: object) -> object:
+    return marker if chat_completion_prompt_cache_breakpoint_for_wire(marker, drop_params=True) == marker else None
+
+
 class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPIConfig):
     def __init__(
         self,
@@ -113,6 +119,13 @@ class BedrockMantleResponsesAPIConfig(BedrockMantleAuthMixin, OpenAIResponsesAPI
             headers["OpenAI-Project"] = litellm_params.aws_bedrock_project_id
         return headers
 
+    def _prompt_cache_breakpoint_to_wire(
+        self, litellm_params: GenericLiteLLMParams, reads_breakpoints: bool, known_support: bool | None
+    ) -> Callable[[object], object]:
+        if reads_breakpoints or known_support is True:
+            return partial(chat_completion_prompt_cache_breakpoint_for_wire, drop_params=True)
+        return _prompt_cache_breakpoint_as_sent_or_none
+
     def supports_native_file_search(self) -> bool:
         return False
 

litellm/llms/cohere/common_utils.py

@@ -3,6 +3,7 @@ from typing import Final, Literal
 
 from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.llms.cohere import CohereV2StreamedToolCall
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import (
     ChatCompletionToolCallChunk,
@@ -222,6 +223,7 @@ class CohereV2ModelResponseIterator:
         self.content_blocks: list = []
         self.tool_index = -1
         self.json_mode = json_mode
+        self.tool_call_indexes_with_arguments: frozenset[int] = frozenset()
 
     def _parse_content_delta(self, chunk: dict) -> str:
         """Parse content-delta chunks to extract text."""
@@ -234,20 +236,23 @@ class CohereV2ModelResponseIterator:
             return content
         return ""
 
-    def _parse_tool_call_delta(self, chunk: dict) -> ChatCompletionToolCallChunk | None:
-        """Parse tool-call-delta chunks to extract tool calls."""
-        delta: Final = chunk.get("delta", {})
-        tool_calls: Final = delta.get("tool_calls", [])
-        if tool_calls:
-            return {
-                "id": tool_calls[0].get("id", ""),
-                "type": "function",
-                "function": {
-                    "name": tool_calls[0].get("name", ""),
-                    "arguments": tool_calls[0].get("arguments", ""),
-                },
-            }
-        return None
+    def _parse_tool_call_delta(self, chunk: dict, index: int) -> ChatCompletionToolCallChunk:
+        """Parse tool-call-start and tool-call-delta chunks to extract tool calls."""
+        tool_call: Final = CohereV2StreamedToolCall.model_validate(chunk)
+        if tool_call.arguments:
+            self.tool_call_indexes_with_arguments = self.tool_call_indexes_with_arguments | {index}
+        return {
+            "id": tool_call.id,
+            "type": "function",
+            "function": {"name": tool_call.name, "arguments": tool_call.arguments},
+            "index": index,
+        }
+
+    def _parse_tool_call_end(self, index: int) -> ChatCompletionToolCallChunk | None:
+        """Close a tool call that streamed no arguments with an empty JSON object, as OpenAI streams one."""
+        if index in self.tool_call_indexes_with_arguments:
+            return None
+        return {"id": None, "type": "function", "function": {"name": None, "arguments": "{}"}, "index": index}
 
     def _parse_tool_plan_delta(self, chunk: dict) -> dict | None:
         """Parse tool-plan-delta events to extract tool plan."""
@@ -301,7 +306,8 @@ class CohereV2ModelResponseIterator:
 
         v2 format:
         - Content: chunk.type == "content-delta" -> chunk.delta.message.content.text
-        - Tool calls: chunk.type == "tool-call-delta" -> chunk.delta.tool_calls
+        - Tool calls: chunk.type in ("tool-call-start", "tool-call-delta") -> chunk.delta.message.tool_calls
+        - Tool call end: chunk.type == "tool-call-end" -> "{}" arguments when the call streamed none
         - Tool plan: chunk.event == "tool-plan-delta" -> chunk.data.delta.message.tool_plan
         - Citations: chunk.event == "citation-start" -> chunk.data.delta.message.citations
         - Finish: chunk.event == "message-end" -> chunk.data.delta.finish_reason
@@ -321,8 +327,12 @@ class CohereV2ModelResponseIterator:
             # Handle different chunk types
             if chunk_type == "content-delta":
                 text = self._parse_content_delta(chunk)
-            elif chunk_type == "tool-call-delta":
-                tool_use = self._parse_tool_call_delta(chunk)
+            elif chunk_type in ("tool-call-start", "tool-call-delta", "tool-call-end"):
+                tool_use = (
+                    self._parse_tool_call_end(index)
+                    if chunk_type == "tool-call-end"
+                    else self._parse_tool_call_delta(chunk, index)
+                )
             elif event_type == "tool-plan-delta":
                 provider_specific_fields = self._parse_tool_plan_delta(chunk)
             elif event_type == "citation-start":

litellm/llms/custom_httpx/aiohttp_handler.py

@@ -14,6 +14,7 @@ from litellm.llms.base_llm.chat.transformation import BaseConfig
 from litellm.llms.base_llm.image_variations.transformation import (
     BaseImageVariationConfig,
 )
+from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import build_pool_wait_trace_config
 from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -90,7 +91,7 @@ class BaseLLMAIOHTTPHandler:
         if self.transport is None:
             connector: Final = self._get_connector()
             if connector:
-                return aiohttp.ClientSession(connector=connector)
+                return aiohttp.ClientSession(connector=connector, trace_configs=[build_pool_wait_trace_config()])
 
         transport: Final = self.transport or self._get_or_create_transport()
         if transport is not None and hasattr(transport, "_get_valid_client_session"):
@@ -99,7 +100,7 @@ class BaseLLMAIOHTTPHandler:
             except RuntimeError:
                 pass
 
-        return aiohttp.ClientSession()
+        return aiohttp.ClientSession(trace_configs=[build_pool_wait_trace_config()])
 
     def _get_async_client_session(self, dynamic_client_session: ClientSession | None = None) -> ClientSession:
         if dynamic_client_session:

litellm/llms/custom_httpx/aiohttp_pool_diagnostics.py

@@ -0,0 +1,130 @@
+import asyncio
+from types import SimpleNamespace
+from typing import Final
+
+from aiohttp import (
+    ClientSession,
+    TraceConfig,
+    TraceConnectionQueuedEndParams,
+    TraceConnectionQueuedStartParams,
+    TraceRequestExceptionParams,
+    TraceRequestRedirectParams,
+    TraceRequestStartParams,
+)
+from yarl import URL
+
+from litellm._logging import verbose_logger
+
+_POOL_WAIT_WARNING_SECONDS: Final = 1.0
+
+
+class _PoolWaitContext(SimpleNamespace):
+    def __init__(self, trace_request_ctx: object = None) -> None:
+        super().__init__()
+        self.host: str | None = None
+        self.port: int | None = None
+        self.started_at: float | None = None
+        self.waited: float = 0.0
+        self.warning: asyncio.TimerHandle | None = None
+        self.warned: bool = False
+
+    def set_destination(self, params: TraceRequestStartParams) -> None:
+        self.host = params.url.host
+        self.port = params.url.port
+
+    def redirect(self, params: TraceRequestRedirectParams) -> None:
+        location: Final = params.response.headers.get("Location") or params.response.headers.get("URI")
+        if location is None:
+            return
+        try:
+            destination: Final = params.url.join(URL(location))
+            host: Final = destination.host
+            port: Final = destination.port
+        except ValueError:
+            return
+        self.host = host
+        self.port = port
+
+    def start(self, session: ClientSession) -> None:
+        loop: Final = asyncio.get_running_loop()
+        self.started_at = loop.time()
+        if not self.warned:
+            self.warning = loop.call_later(max(0.0, _POOL_WAIT_WARNING_SECONDS - self.waited), self.warn, session)
+
+    def elapsed(self) -> float:
+        return self.waited + (
+            asyncio.get_running_loop().time() - self.started_at if self.started_at is not None else 0.0
+        )
+
+    def warn(self, session: ClientSession) -> None:
+        if self.warned or self.started_at is None:
+            return
+        self.warned = True
+        connector: Final = session.connector
+        if connector is None:
+            return
+        verbose_logger.warning(
+            "aiohttp connection pool saturated: host=%s port=%s waited=%.3fs for a connection slot "
+            "(acquired_total=%s, limit=%s, limit_per_host=%s). "
+            "Wait occurred before sending the request upstream.",
+            self.host,
+            self.port,
+            self.elapsed(),
+            len(connector._acquired),  # pyright: ignore[reportPrivateUsage]  # aiohttp exposes no public acquired count
+            connector.limit,
+            connector.limit_per_host,
+        )
+
+    def finish(self, session: ClientSession, failed: bool = False) -> None:
+        if self.warning is not None:
+            self.warning.cancel()
+            self.warning = None
+        if self.started_at is None:
+            return
+        if failed or self.elapsed() >= _POOL_WAIT_WARNING_SECONDS:
+            self.warn(session)
+        self.waited = self.elapsed()
+        self.started_at = None
+
+
+async def _on_request_start(session: ClientSession, context: SimpleNamespace, params: TraceRequestStartParams) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.set_destination(params)
+
+
+async def _on_queued_start(
+    session: ClientSession, context: SimpleNamespace, params: TraceConnectionQueuedStartParams
+) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.start(session)
+
+
+async def _on_queued_end(
+    session: ClientSession, context: SimpleNamespace, params: TraceConnectionQueuedEndParams
+) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.finish(session)
+
+
+async def _on_request_exception(
+    session: ClientSession, context: SimpleNamespace, params: TraceRequestExceptionParams
+) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.finish(session, failed=True)
+
+
+async def _on_request_redirect(
+    session: ClientSession, context: SimpleNamespace, params: TraceRequestRedirectParams
+) -> None:
+    if isinstance(context, _PoolWaitContext):
+        context.redirect(params)
+
+
+def build_pool_wait_trace_config() -> TraceConfig:
+    trace: Final = TraceConfig(trace_config_ctx_factory=_PoolWaitContext)
+    trace.on_request_start.append(_on_request_start)
+    trace.on_connection_queued_start.append(_on_queued_start)
+    trace.on_connection_queued_end.append(_on_queued_end)
+    trace.on_request_exception.append(_on_request_exception)
+    trace.on_request_redirect.append(_on_request_redirect)
+    return trace

litellm/llms/custom_httpx/http_handler.py

@@ -44,6 +44,7 @@ from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
 from litellm.litellm_core_utils.request_timeout_resolver import (
     get_configured_request_timeout,
 )
+from litellm.llms.custom_httpx.aiohttp_pool_diagnostics import build_pool_wait_trace_config
 from litellm.llms.custom_httpx.http2_connection_spread import AsyncHTTP2ConnectionSpread
 from litellm.types.llms.custom_http import *
 
@@ -1456,6 +1457,7 @@ class AsyncHTTPHandler:
                 connector=TCPConnector(**transport_connector_kwargs),
                 cookie_jar=DummyCookieJar(),
                 trust_env=trust_env,
+                trace_configs=[build_pool_wait_trace_config()],
             )
 
         # Use shared session if provided and valid

litellm/llms/custom_httpx/llm_http_handler.py

@@ -131,6 +131,8 @@ from litellm.responses.streaming_iterator import (
     ResponsesAPIStreamingIterator,
     ResponsesWebSocketStreaming,
     SyncResponsesAPIStreamingIterator,
+    TokenLimitCallback,
+    with_token_limit_checks,
 )
 from litellm.types.containers.main import (
     ContainerFileListResponse,
@@ -598,10 +600,10 @@ def _decoded_body_headers(response: httpx.Response) -> httpx.Headers:
     )
 
 
-def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
-    """Duck-type discover proxy hooks exposing per-frame project ITPM/OTPM
-    enforcement, so the Responses WebSocket loop can charge every
-    ``response.create`` frame, not just the connection's first one.
+def _ws_callbacks_exposing(method_name: str) -> tuple[object, ...]:
+    """Duck-type discover proxy hooks exposing a per-frame check, so the
+    Responses WebSocket loop can run it on every ``response.create`` frame,
+    not just the connection's first one.
 
     Uses duck-typing on ``litellm.callbacks`` (rather than importing the
     proxy hook directly) to avoid a layering violation (SDK importing from
@@ -612,13 +614,28 @@ def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
     callbacks: Final = cast(  # cast-ok: callback registry is inspected before protocol use
         Sequence[object], _litellm.callbacks
     )
+    return tuple(callback for callback in callbacks if callable(getattr(callback, method_name, None)))
+
+
+def _collect_ws_project_quota_callbacks() -> tuple[ProjectQuotaCallback, ...]:
     return tuple(
         cast(ProjectQuotaCallback, callback)  # cast-ok: required callback method is callable
-        for callback in callbacks
-        if callable(getattr(callback, "enforce_project_io_token_quota_for_frame", None))
+        for callback in _ws_callbacks_exposing("enforce_project_io_token_quota_for_frame")
     )
 
 
+def _collect_ws_token_limit_callbacks() -> tuple[TokenLimitCallback, ...]:
+    return tuple(
+        cast(TokenLimitCallback, callback)  # cast-ok: required callback method is callable
+        for callback in _ws_callbacks_exposing("enforce_token_limits_for_frame")
+    )
+
+
+def _ws_model_group(litellm_metadata: Mapping[str, object] | None, model: str) -> str:
+    model_group: Final = (litellm_metadata or {}).get("model_group")
+    return model_group if isinstance(model_group, str) and model_group else model
+
+
 def _unwrap_provider_error(error: Exception) -> Exception:
     if not isinstance(error, httpx.TransportError):
         return error
@@ -7285,6 +7302,12 @@ class BaseLLMHTTPHandler:
         - Forwards events over the websocket connection
         """
         _ws_quota_callbacks: Final = _collect_ws_project_quota_callbacks()
+        _ws_turn_admission: Final = with_token_limit_checks(
+            turn_admission,
+            _collect_ws_token_limit_callbacks(),
+            user_api_key_dict,
+            _ws_model_group(litellm_metadata, model),
+        )
 
         _ws_guardrail_callbacks: list = []
         _ws_output_guardrail_callbacks: list = []
@@ -7332,7 +7355,7 @@ class BaseLLMHTTPHandler:
                 custom_llm_provider=custom_llm_provider,
                 first_message=first_message,
                 quota_callbacks=_ws_quota_callbacks,
-                turn_admission=turn_admission,
+                turn_admission=_ws_turn_admission,
                 guardrail_callbacks=_ws_guardrail_callbacks,  # pyright: ignore[reportUnknownArgumentType]  # duck-typed callbacks from litellm.callbacks
                 frame_guard=frame_guard,
                 **kwargs,
@@ -7429,7 +7452,7 @@ class BaseLLMHTTPHandler:
                     authorized_model=model,
                     custom_llm_provider=custom_llm_provider,
                     request_defaults=request_defaults,
-                    turn_admission=turn_admission,
+                    turn_admission=_ws_turn_admission,
                     frame_guard=frame_guard,
                 )
                 return await streaming.bidirectional_forward()

litellm/llms/deepgram/common_utils.py

@@ -1,16 +1,21 @@
+import json
 import math
+import re
 from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Final
-from urllib.parse import parse_qs, urlparse
+from urllib.parse import parse_qs, urlparse, urlsplit, urlunsplit
 
 import httpx
 
 import litellm
-from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL
+from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL, DEEPGRAM_SPEAK_DEFAULT_MODEL
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.passthrough_endpoints.pass_through_endpoints import WebSocketSessionUsage
 from litellm.types.utils import LlmProviders
+from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration
 
 _WEBSOCKET_SCHEMES: Final = MappingProxyType({"https": "wss", "http": "ws", "wss": "wss", "ws": "ws"})
 DEEPGRAM_LISTEN_CALLBACK_PARAMS: Final = frozenset({"callback", "callback_method"})
@@ -199,3 +204,71 @@ def deepgram_listen_transcript(websocket_messages: Sequence[Mapping[str, object]
         if frame.get("type") == "Results"
         if (transcript := _final_transcript(frame)) is not None
     )
+
+
+_AURA_GENERATION: Final = re.compile(r"aura(?:-\d+)?(?=-[a-z])")
+
+
+def deepgram_speak_websocket_target(api_base: str | None, query_string: str) -> str:
+    speak_url: Final = urlsplit(join_api_base(api_base or DEEPGRAM_DEFAULT_API_BASE, "speak"))
+    scheme: Final = _WEBSOCKET_SCHEMES.get(speak_url.scheme, speak_url.scheme)
+    params: Final = _first_occurrences(query_string)
+    query: Final = params if params.get("model") else params.remove("model").add("model", DEEPGRAM_SPEAK_DEFAULT_MODEL)
+    return urlunsplit((scheme, speak_url.netloc, speak_url.path, str(query), ""))
+
+
+def deepgram_speak_model(upstream_url: str) -> str:
+    models: Final = parse_qs(urlparse(upstream_url).query).get("model")
+    return models[0] if models else DEEPGRAM_SPEAK_DEFAULT_MODEL
+
+
+def deepgram_speak_requested_model(query_string: str) -> str:
+    """Read off the upstream target, so the voice a key is authorized for is the voice Deepgram receives"""
+    return deepgram_speak_model(deepgram_speak_websocket_target(None, query_string))
+
+
+def deepgram_speak_registry_key(upstream_url: str) -> str:
+    """Deepgram bills every voice of an Aura generation at one rate, so ``aura-2-thalia-en`` is priced at the
+    ``deepgram/aura-2`` row"""
+    model: Final = deepgram_speak_model(upstream_url)
+    generation: Final = _AURA_GENERATION.match(model)
+    return f"{LlmProviders.DEEPGRAM.value}/{generation.group() if generation else model}"
+
+
+def deepgram_speak_character_rate(upstream_url: str) -> float | None:
+    """Only an exact registry hit counts, so a voice is never billed at a neighbouring row's rate"""
+    registry_key: Final = deepgram_speak_registry_key(upstream_url)
+    model_info: Final = lookup_model_info_for_registration(registry_key, LlmProviders.DEEPGRAM.value)
+    if isinstance(model_info, ModelInfoLookupMiss) or model_info["key"] != registry_key:
+        return None
+    return model_info.get("input_cost_per_character")
+
+
+@dataclass(frozen=True, slots=True)
+class DeepgramSpeakMeter:
+    """Holds the rate checked when the socket opened, so a cost-map reload mid-session can't change or drop the price"""
+
+    upstream_url: str
+    character_rate: float
+
+    def measure(self, client_text: str) -> int:
+        """Counts a Speak message's text as it is sent. Deepgram also speaks, and bills, unflushed text on its own (a
+        long buffer, Close, a close frame), so counting only flushed text would let a client get audio free, while
+        counting on send can only over-bill text Deepgram discards, such as a Clear or an oversized Speak. Never
+        raises, so a malformed frame can't end the session"""
+        try:
+            message: Final[object] = json.loads(client_text)
+        except (ValueError, RecursionError):
+            return 0
+        match message:
+            case {"type": "Speak", "text": str() as text}:
+                return len(text)
+            case _:
+                return 0
+
+    def price(self, total: int) -> WebSocketSessionUsage:
+        return WebSocketSessionUsage(
+            model=deepgram_speak_model(self.upstream_url),
+            custom_llm_provider=LlmProviders.DEEPGRAM.value,
+            response_cost=total * self.character_rate,
+        )

litellm/llms/gemini/common_utils.py

@@ -537,13 +537,16 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
     ) -> TokenCountResponse | None:
         import copy
 
+        from litellm.llms.anthropic.messages.context_management.inference_view import (
+            InferenceRequestViewError,
+            count_context_edits,
+            inference_request_view,
+        )
         from litellm.llms.gemini.count_tokens.handler import GoogleAIStudioTokenCounter
         from litellm.llms.vertex_ai.count_tokens.context import (
             VertexGeminiCountContextError,
-            count_context_edits,
             count_context_error_response,
             gemini_count_contents,
-            inference_request_view,
             malformed_tool_result_text_count_error,
             provider_error_response,
             unconvertible_messages_error,
@@ -556,7 +559,7 @@ class GoogleAIStudioTokenCounter(BaseTokenCounter):
             messages_format,
             count_context_edits(context_management, tools, (deployment or {}).get("litellm_params", {}), model_to_use),
         )
-        if isinstance(request_view, VertexGeminiCountContextError):
+        if isinstance(request_view, InferenceRequestViewError):
             return count_context_error_response(request_view, request_model, model_to_use)
         count_context: Final = vertex_gemini_count_context(system=request_view.system, tools=tools, model=model_to_use)
         if isinstance(count_context, VertexGeminiCountContextError):

litellm/llms/langgraph/chat/sse_iterator.py

@@ -41,8 +41,9 @@ class LangGraphSSEStreamIterator:
         return self
 
     def __aiter__(self):
-        """Initialize async iteration."""
-        self.async_line_iterator = aiter_sse_lines(self.response.aiter_text())
+        """Initialize async iteration once; CustomStreamWrapper re-enters `async for` on every chunk."""
+        if self.async_line_iterator is None:
+            self.async_line_iterator = aiter_sse_lines(self.response.aiter_text())
         return self
 
     def _parse_sse_line(self, line: str) -> ModelResponseStream | None:

litellm/llms/litellm_proxy/skills/skill_search.py

@@ -131,7 +131,7 @@ async def search_skills(
     user_api_key_dict: UserAPIKeyAuth,
     proxy_logging_obj: ProxyLogging,
 ) -> SkillSearchOutcome:
-    if embedding_model is None:
+    if embedding_model is None or not embedding_model.strip():
         return SkillSearchNotConfigured(
             reason="skill search needs litellm_settings.skill_search_embedding_model set to an embedding model from model_list"
         )

litellm/llms/oci/chat/transformation.py

@@ -20,7 +20,11 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
 from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
-from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    merge_consecutive_system_messages,
+    messages_with_file_parts_as_image_urls,
+    system_messages_first,
+)
 from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
 from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@@ -693,14 +697,18 @@ class OCIChatConfig(BaseConfig):
                 chatRequest=chat_request,
             )
         else:
+            generic_messages: Final = list(messages_with_file_parts_as_image_urls(messages))
+            normalized_messages: Final = (
+                merge_consecutive_system_messages(system_messages_first(generic_messages))
+                if model.lower().removeprefix("oci/").startswith("google.gemini")
+                else generic_messages
+            )
             data = OCICompletionPayload(
                 compartmentId=oci_compartment_id,
                 servingMode=serving_mode,
                 chatRequest=OCIChatRequestPayload(
                     apiFormat=vendor.value,
-                    messages=adapt_messages_to_generic_oci_standard(
-                        list(messages_with_file_parts_as_image_urls(messages))
-                    ),
+                    messages=adapt_messages_to_generic_oci_standard(normalized_messages),
                     **self._get_optional_params(vendor, optional_params, model),
                 ),
             )

litellm/llms/openai/chat/gpt_transformation.py

@@ -21,6 +21,7 @@ from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response impo
     should_convert_tool_call_to_json_mode,
 )
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    chat_completion_prompt_cache_breakpoints_for_wire,
     drop_non_python_regex_patterns,
     drop_tool_reference_parts_from_tool_messages,
     flatten_combinators_and_drop_non_python_regex_patterns,
@@ -556,9 +557,10 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         """
         A deployment that reads OpenAI's ``prompt_cache_breakpoint`` (gpt-5.6+ on
         api.openai.com, or one opted in through ``prompt_cache_options`` or the cost
-        map) gets each Anthropic ``cache_control`` mark rewritten in place. Any other
-        endpoint keeps or strips the marks as before, and under drop_params also loses
-        every ``prompt_cache_breakpoint`` it cannot read.
+        map) gets each Anthropic ``cache_control`` mark rewritten in place, and under
+        drop_params each content part's ``prompt_cache_breakpoint`` cut down to what chat
+        completions accepts. Any other endpoint keeps or strips the marks as before, and
+        under drop_params also loses every ``prompt_cache_breakpoint`` it cannot read.
         """
         from litellm.integrations.anthropic_cache_control_hook import AnthropicCacheControlHook
 
@@ -572,9 +574,15 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
                 _claude_messages_within_cache_control_cap(model, messages, tools, litellm_params, request_params),
                 tools,
             )
-        drops_breakpoints: Final = not reads_breakpoints and _drops_params(litellm_params)
+        drop_params: Final = _drops_params(litellm_params)
+        drops_breakpoints: Final = not reads_breakpoints and drop_params
         renamed: Final = (
-            [AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(message) for message in messages]
+            [
+                AnthropicCacheControlHook.cache_control_as_prompt_cache_breakpoints(
+                    chat_completion_prompt_cache_breakpoints_for_wire(message, drop_params)
+                )
+                for message in messages
+            ]
             if reads_breakpoints
             else [without_prompt_cache_breakpoints(message) for message in messages]
             if drops_breakpoints

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -38,8 +38,13 @@ from itertools import accumulate, chain, groupby, repeat
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Union, cast
 
+from openai.types.responses.response_output_refusal import ResponseOutputRefusal
 from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
-from typing_extensions import ReadOnly, TypedDict
+from typing_extensions import (
+    ReadOnly,
+    TypedDict,
+    TypeIs,  # noqa: TID251  # TypeIs reaches typing only on 3.13
+)
 
 from litellm._logging import verbose_proxy_logger
 from litellm.completion_extras.litellm_responses_transformation.transformation import (
@@ -99,6 +104,7 @@ from litellm.types.llms.openai import (
     OutputTextDeltaEvent,
     OutputTextDoneEvent,
     ResponseAPIUsage,
+    ResponseCompactionItem,
     ResponseCompletedEvent,
     ResponsesAPIOptionalRequestParams,
     ResponsesAPIResponse,
@@ -277,6 +283,7 @@ _TOOL_CALL_PAYLOAD_EVENT_TYPES: Final = _TOOL_CALL_PAYLOAD_DELTA_EVENT_TYPES | f
 )
 _OUTPUT_ITEM_EVENT_TYPES: Final = frozenset({"response.output_item.added", "response.output_item.done"})
 _OUTPUT_TEXT_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.output_text.done"})
+_TEXT_DELTA_EVENT_TYPES: Final = frozenset({"response.output_text.delta", "response.refusal.delta"})
 _TOOL_OUTPUT_TEXT_PART_TYPES: Final = frozenset({"input_text", "output_text", "text"})
 _PATCHABLE_ITEM_FIELDS: Final[Mapping[str, str]] = MappingProxyType(
     {"function_call_output": "output", "custom_tool_call_output": "output", "message": "content"}
@@ -537,6 +544,14 @@ def _scanned_text_field(item: Mapping[str, object]) -> str:
     return "output" if item.get("type") in TOOL_CALL_OUTPUT_ITEM_TYPES else "content"
 
 
+def _content_part_text_field(part: object) -> str:
+    return "refusal" if stream_item_field(part, "type") == "refusal" else "text"
+
+
+def _has_writable_field(part: object, field: str) -> bool:
+    return isinstance(part, dict) or hasattr(part, field)
+
+
 def _tool_output_part_text(part: object) -> str | None:
     if not isinstance(part, dict):
         return None
@@ -702,6 +717,15 @@ def _next_stream_sequence_number(responses_so_far: Sequence[object] | None) -> i
     return next((n + 1 for n in sequence_numbers if isinstance(n, int)), 0)
 
 
+def _is_responses_input(
+    input_data: object,
+) -> TypeIs["str | ResponseInputParam"]:  # guard-ok: str, or a list of Mapping items
+    if isinstance(input_data, str):
+        return True
+    items: Final = cast(list[object], input_data)  # cast-ok: isinstance narrows to list[Unknown] only
+    return isinstance(input_data, list) and all(isinstance(item, (Mapping, ResponseCompactionItem)) for item in items)
+
+
 class OpenAIResponsesHandler(BaseTranslation):
     """
     Handler for processing OpenAI Responses API with guardrails.
@@ -723,8 +747,8 @@ class OpenAIResponsesHandler(BaseTranslation):
         Transforms `input` (string or ResponseInputParam) and optional
         `instructions` into chat completion messages.
         """
-        input_data: Final = data.get("input")
-        if input_data is None:
+        input_data: Final[object] = data.get("input")
+        if not _is_responses_input(input_data):
             return None
         messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
             input=input_data,
@@ -993,7 +1017,7 @@ class OpenAIResponsesHandler(BaseTranslation):
             for content_idx, content_item in enumerate(content):
                 if isinstance(content_item, dict):
                     # Extract text
-                    text_str = content_item.get("text", None)
+                    text_str = content_item.get(_content_part_text_field(content_item), None)
                     if text_str is not None:
                         texts_to_check.append(text_str)
                         task_mappings.append((msg_idx, int(content_idx)))
@@ -1042,8 +1066,7 @@ class OpenAIResponsesHandler(BaseTranslation):
 
             elif isinstance(content, list):
                 # Replace specific text item in list content
-                if isinstance(messages[msg_idx][field][content_idx_optional], dict):
-                    messages[msg_idx][field][content_idx_optional]["text"] = guardrail_response
+                self._write_content_part_text(content[content_idx_optional], guardrail_response)
 
     async def process_output_response(
         self,
@@ -1380,7 +1403,8 @@ class OpenAIResponsesHandler(BaseTranslation):
             if isinstance(output_index, int) and isinstance(content_index, int)
         )
         if len(placeable_positions) != len(scanned_positions) or any(
-            stream_item_field(event, "type") not in _OUTPUT_TEXT_EVENT_TYPES for event in scanned_events
+            stream_item_field(event, "type") not in _OUTPUT_TEXT_EVENT_TYPES | _TEXT_DELTA_EVENT_TYPES
+            for event in scanned_events
         ):
             from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite
 
@@ -1401,6 +1425,12 @@ class OpenAIResponsesHandler(BaseTranslation):
         else:
             setattr(event, field, value)
 
+    @staticmethod
+    def _write_content_part_text(part: object, text: str) -> None:
+        field: Final = _content_part_text_field(part)
+        if _has_writable_field(part, field):
+            OpenAIResponsesHandler._write_event_field(part, field, text)
+
     @staticmethod
     def _write_compaction_summary(item: object, summary: str) -> None:
         encrypted_content: Final = stream_item_field(item, "encrypted_content")
@@ -1416,11 +1446,12 @@ class OpenAIResponsesHandler(BaseTranslation):
     ) -> None:
         """Sync pre-completion stream events with the rewritten completed
         response, keyed by ``(output_index, content_index)``: the first
-        ``output_text.delta`` for a rewritten item carries the full rewritten
-        text and the rest are blanked, while ``output_text.done``,
-        ``content_part.done``, and ``output_item.done`` events carry the full
-        rewritten text, so every event a client may read agrees with the
-        rewritten ``response.completed`` payload."""
+        ``output_text.delta`` or ``refusal.delta`` for a rewritten item carries
+        the full rewritten text and the rest are blanked, while
+        ``output_text.done``, ``refusal.done``, ``content_part.done``, and
+        ``output_item.done`` events carry the full rewritten text, so every
+        event a client may read agrees with the rewritten ``response.completed``
+        payload."""
         delta_replacements: Final = MappingProxyType(
             {position: chain((rewritten,), repeat("")) for position, rewritten in rewrites_by_position.items()}
         )
@@ -1436,14 +1467,14 @@ class OpenAIResponsesHandler(BaseTranslation):
             if not isinstance(output_index, int) or not isinstance(content_index, int):
                 continue
             position = (output_index, content_index)
-            if event_type == "response.output_text.delta" and position in delta_replacements:
+            if event_type in _TEXT_DELTA_EVENT_TYPES and position in delta_replacements:
                 self._write_event_field(event, "delta", next(delta_replacements[position]))
             elif event_type == "response.output_text.done" and position in rewrites_by_position:
                 self._write_event_field(event, "text", rewrites_by_position[position])
+            elif event_type == "response.refusal.done" and position in rewrites_by_position:
+                self._write_event_field(event, "refusal", rewrites_by_position[position])
             elif event_type == "response.content_part.done" and position in rewrites_by_position:
-                part = event.get("part")
-                if isinstance(part, dict) or hasattr(part, "text"):
-                    self._write_event_field(part, "text", rewrites_by_position[position])
+                self._write_content_part_text(event.get("part"), rewrites_by_position[position])
 
     @staticmethod
     def _sync_output_item_done_event(
@@ -1459,7 +1490,7 @@ class OpenAIResponsesHandler(BaseTranslation):
         for (item_idx, content_idx), rewritten in rewrites_by_position.items():
             if item_idx != output_index or not 0 <= content_idx < len(content):
                 continue
-            OpenAIResponsesHandler._write_event_field(content[content_idx], "text", rewritten)
+            OpenAIResponsesHandler._write_content_part_text(content[content_idx], rewritten)
 
     def _deliver_ended_stream_tool_call_rewrites(
         self,
@@ -1814,7 +1845,7 @@ class OpenAIResponsesHandler(BaseTranslation):
             return
 
         # Handle both GenericResponseOutputItem and dict
-        content: list[OutputText] | list[dict] | None = None
+        content: list[OutputText | ResponseOutputRefusal] | list[dict[str, object]] | None = None
         if isinstance(output_item, BaseModel):
             try:
                 output_item_dump: Final = output_item.model_dump()
@@ -1837,17 +1868,9 @@ class OpenAIResponsesHandler(BaseTranslation):
 
         verbose_proxy_logger.debug("OpenAI Responses API: Processing output item: %s", output_item)
 
-        # Iterate through content items (list of OutputText objects)
         for content_idx, content_item in enumerate(content):
-            # Handle both OutputText objects and dicts
-            if isinstance(content_item, OutputText):
-                text_content = content_item.text
-            elif isinstance(content_item, dict):
-                text_content = content_item.get("text")
-            else:
-                continue
-
-            if text_content:
+            text_content = stream_item_field(content_item, _content_part_text_field(content_item))
+            if isinstance(text_content, str) and text_content:
                 texts_to_check.append(text_content)
                 task_mappings.append((output_idx, int(content_idx)))
 
@@ -1887,33 +1910,24 @@ class OpenAIResponsesHandler(BaseTranslation):
             # Handle both GenericResponseOutputItem, BaseModel, and dict
             if isinstance(output_item, GenericResponseOutputItem):
                 if output_item.content and content_idx < len(output_item.content):
-                    content_item = output_item.content[content_idx]
-                    if isinstance(content_item, OutputText):
-                        content_item.text = guardrail_response
-                    elif isinstance(content_item, dict):
-                        content_item["text"] = guardrail_response
+                    self._write_content_part_text(output_item.content[content_idx], guardrail_response)
             elif isinstance(output_item, BaseModel):
                 # Handle other Pydantic models by converting to GenericResponseOutputItem
                 try:
                     generic_item = GenericResponseOutputItem.model_validate(output_item.model_dump())
-                    if generic_item.content and content_idx < len(generic_item.content):
-                        content_item = generic_item.content[content_idx]
-                        if isinstance(content_item, OutputText):
-                            content_item.text = guardrail_response
-                            # Update the original response output
-                            if hasattr(output_item, "content") and output_item.content:
-                                original_content = output_item.content[content_idx]
-                                if hasattr(original_content, "text"):
-                                    original_content.text = guardrail_response
+                    if (
+                        generic_item.content
+                        and content_idx < len(generic_item.content)
+                        and hasattr(output_item, "content")
+                        and output_item.content
+                    ):
+                        self._write_content_part_text(output_item.content[content_idx], guardrail_response)
                 except Exception:
                     pass
             elif isinstance(output_item, dict):
                 content = output_item.get("content", [])
                 if content and content_idx < len(content):
-                    if isinstance(content[content_idx], dict):
-                        content[content_idx]["text"] = guardrail_response
-                    elif hasattr(content[content_idx], "text"):
-                        content[content_idx].text = guardrail_response
+                    self._write_content_part_text(content[content_idx], guardrail_response)
 
     def build_block_sse_chunks(
         self,
@@ -1974,7 +1988,7 @@ class OpenAIResponsesHandler(BaseTranslation):
             "annotations": (),
             "logprobs": None,
         }
-        return (
+        events: Final = (
             *_open_item_closing_events(responses_so_far),
             OutputItemAddedEvent(
                 type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED,
@@ -2019,6 +2033,10 @@ class OpenAIResponsesHandler(BaseTranslation):
                 response=_blocked_response(exc, response_id=response_id, model=model, output_item=item),
             ),
         )
+        return tuple(
+            event.model_copy(update={"sequence_number": number})
+            for number, event in enumerate(events, start=_next_stream_sequence_number(responses_so_far))
+        )
 
 
 class _BlockedContentPart(TypedDict):

litellm/llms/openai/responses/transformation.py

@@ -1,5 +1,5 @@
 from collections.abc import Callable, Collection, Iterable, Mapping, Sequence
-from functools import lru_cache
+from functools import lru_cache, partial
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Protocol, cast, get_type_hints
 
@@ -12,7 +12,7 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.integrations.anthropic_cache_control_hook import (
     AnthropicCacheControlHook,
-    openai_prompt_cache_breakpoint_support,
+    deployment_prompt_cache_breakpoint_support,
     targets_openai_api,
 )
 from litellm.litellm_core_utils.core_helpers import normalize_drop_params, process_response_headers
@@ -25,6 +25,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     drop_non_python_regex_patterns,
     flatten_combinators_and_drop_non_python_regex_patterns,
     prompt_cache_breakpoint_for_wire,
+    whole_prompt_cache_breakpoint_or_none,
     with_prompt_cache_breakpoint,
 )
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
@@ -103,13 +104,6 @@ _INPUT_ITEM: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str,
 _CONTENT_PARTS: Final[TypeAdapter[Sequence[Mapping[str, object]]]] = TypeAdapter(Sequence[Mapping[str, object]])
 
 
-class _DeclaredModel(BaseModel):
-    model_config = ConfigDict(frozen=True, extra="ignore", from_attributes=True)
-
-    base_model: object = None
-    model_info: object = None
-
-
 class _DeploymentHost(BaseModel):
     """`responses()` builds the params from raw kwargs, where callers may spell the host `base_url`."""
 
@@ -241,12 +235,12 @@ def _json_array_or_empty(value: object) -> Sequence[object]:
     return value if isinstance(value, (list, tuple)) else ()
 
 
-def _item_with_wire_safe_prompt_cache_breakpoints(item: object, keeps_markers: bool) -> object:
+def _item_with_wire_safe_prompt_cache_breakpoints(item: object, to_wire: Callable[[object], object] | None) -> object:
     fields: Final = _json_object_or_none(item)
     if fields is None or not _carries_prompt_cache_breakpoint(fields):
         return item
     wire_safe_fields: Final = {
-        key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, keeps_markers)
+        key: _parts_with_wire_safe_prompt_cache_breakpoints(key, value, to_wire)
         for key, value in fields.items()
         if key != "prompt_cache_breakpoint"
     }
@@ -270,21 +264,23 @@ def _parts_carry_prompt_cache_breakpoint(parts: object) -> bool:
     return any(isinstance(part, Mapping) and "prompt_cache_breakpoint" in part for part in _json_array_or_empty(parts))
 
 
-def _parts_with_wire_safe_prompt_cache_breakpoints(key: str, value: object, keeps_markers: bool) -> object:
+def _parts_with_wire_safe_prompt_cache_breakpoints(
+    key: str, value: object, to_wire: Callable[[object], object] | None
+) -> object:
     if key not in _INPUT_PART_LIST_KEYS or not _parts_carry_prompt_cache_breakpoint(value):
         return value
     parts: Final = _json_array_or_empty(value)
-    wire_safe_parts: Final = tuple(_part_with_wire_safe_prompt_cache_breakpoint(part, keeps_markers) for part in parts)
+    wire_safe_parts: Final = tuple(_part_with_wire_safe_prompt_cache_breakpoint(part, to_wire) for part in parts)
     return value if _same_objects(wire_safe_parts, parts) else list(wire_safe_parts)
 
 
-def _part_with_wire_safe_prompt_cache_breakpoint(part: object, keeps_markers: bool) -> object:
+def _part_with_wire_safe_prompt_cache_breakpoint(part: object, to_wire: Callable[[object], object] | None) -> object:
     fields: Final = _json_object_or_none(part)
     if fields is None or "prompt_cache_breakpoint" not in fields:
         return part
     marker: Final = fields["prompt_cache_breakpoint"]
-    wire_marker: Final = prompt_cache_breakpoint_for_wire(marker, drop_params=True) if keeps_markers else None
-    if keeps_markers and wire_marker == marker:
+    wire_marker: Final = None if to_wire is None else to_wire(marker)
+    if to_wire is not None and wire_marker == marker:
         return part
     unmarked: Final = {key: value for key, value in fields.items() if key != "prompt_cache_breakpoint"}
     return with_prompt_cache_breakpoint(unmarked, wire_marker)
@@ -681,24 +677,31 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
     ) -> str | ResponseInputParam:
         if not isinstance(input, list) or not drops_params(litellm_params):
             return input
-        keeps_markers: Final = reads_breakpoints or self._keeps_prompt_cache_breakpoints(
-            model=model, litellm_params=litellm_params
-        )
-        wire_safe_items: Final = tuple(
-            _item_with_wire_safe_prompt_cache_breakpoints(item, keeps_markers) for item in input
+        known_support: Final = self._prompt_cache_breakpoint_support(model=model, litellm_params=litellm_params)
+        to_wire: Final = (
+            self._prompt_cache_breakpoint_to_wire(litellm_params, reads_breakpoints, known_support)
+            if reads_breakpoints or known_support is not False
+            else None
         )
+        wire_safe_items: Final = tuple(_item_with_wire_safe_prompt_cache_breakpoints(item, to_wire) for item in input)
         if _same_objects(wire_safe_items, input):
             return input
         wire_safe_input: Final = list(wire_safe_items)
         return cast("ResponseInputParam", wire_safe_input)  # cast-ok: items keep their shape, minus a rejected marker
 
-    def _keeps_prompt_cache_breakpoints(self, model: str, litellm_params: GenericLiteLLMParams) -> bool:
-        provider: Final = _provider_name(self.custom_llm_provider)
-        base_model: Final = self._declared_base_model(model=model, litellm_params=litellm_params)
-        known_support: Final = (
-            openai_prompt_cache_breakpoint_support(name, provider) for name in dict.fromkeys((base_model, model))
+    def _prompt_cache_breakpoint_to_wire(
+        self, litellm_params: GenericLiteLLMParams, reads_breakpoints: bool, known_support: bool | None
+    ) -> Callable[[object], object]:
+        opted_in_openai_host: Final = reads_breakpoints and self.custom_llm_provider == LlmProviders.OPENAI
+        known_openai_reader: Final = known_support is True and self._targets_openai_api(litellm_params)
+        if opted_in_openai_host or known_openai_reader:
+            return partial(prompt_cache_breakpoint_for_wire, drop_params=True)
+        return whole_prompt_cache_breakpoint_or_none
+
+    def _prompt_cache_breakpoint_support(self, model: str, litellm_params: GenericLiteLLMParams) -> bool | None:
+        return deployment_prompt_cache_breakpoint_support(
+            model, _provider_name(self.custom_llm_provider), litellm_params
         )
-        return next((support for support in known_support if support is not None), None) is not False
 
     def _targets_openai_api(self, litellm_params: GenericLiteLLMParams) -> bool:
         host: Final = _DeploymentHost.model_validate(litellm_params)
@@ -872,13 +875,6 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         base_model: Final[object] = model_info.get("base_model") if isinstance(model_info, dict) else None
         return base_model if isinstance(base_model, str) and base_model else model
 
-    @staticmethod
-    def _declared_base_model(model: str, litellm_params: GenericLiteLLMParams) -> str:
-        declared: Final = _DeclaredModel.model_validate(litellm_params)
-        model_info: Final = _json_object_or_none(declared.model_info)
-        candidates: Final = (declared.base_model, None if model_info is None else model_info.get("base_model"))
-        return next((name for name in candidates if isinstance(name, str) and name), model)
-
     @staticmethod
     def _sanitized_tool_entry(
         entry: Mapping[str, object],

litellm/llms/vertex_ai/batches/transformation.py

@@ -129,6 +129,17 @@ def _native_vertex_row_usage(
     return calculate_usage(completion_response)
 
 
+def native_vertex_batch_row_model(row: Mapping[str, object], model_name: str | None) -> str | None:
+    response: Final = as_json_object(row.get("response"))
+    model_version: Final = (
+        response.get("model" if is_vertex_anthropic_batch_output_response(response) else "modelVersion")
+        if response is not None
+        else None
+    )
+    deployment_model: Final = model_name if model_name and "*" not in model_name else None
+    return deployment_model or (model_version if isinstance(model_version, str) else model_name)
+
+
 def native_vertex_batch_row_stats(
     row: Mapping[str, object],
     model_name: str | None,
@@ -156,11 +167,7 @@ def native_vertex_batch_row_stats(
     if usage is None:
         return None
     total_tokens: Final = usage.total_tokens or (usage.prompt_tokens + usage.completion_tokens)
-    model_version: Final = response_body.get(
-        "model" if is_vertex_anthropic_batch_output_response(response_body) else "modelVersion"
-    )
-    deployment_model: Final = model_name if model_name and "*" not in model_name else None
-    model: Final = deployment_model or (model_version if isinstance(model_version, str) else model_name)
+    model: Final = native_vertex_batch_row_model(row, model_name)
     if model is None:
         verbose_logger.warning(
             "vertex_ai batch output row could not be costed, so it is billed at $0 and the rest of the batch "

litellm/llms/vertex_ai/common_utils.py

@@ -1458,6 +1458,7 @@ class VertexAITokenCounter(BaseTokenCounter):
     ) -> TokenCountResponse | None:
         import copy
 
+        from litellm.llms.anthropic.common_utils import context_management_with_edits
         from litellm.llms.vertex_ai.vertex_ai_partner_models.count_tokens.handler import count_tokens_publisher
         from litellm.llms.vertex_ai.vertex_ai_partner_models.main import (
             VertexAIError as PartnerVertexAIError,
@@ -1539,6 +1540,9 @@ class VertexAITokenCounter(BaseTokenCounter):
                     system=system,
                     tools=tools,
                     count_options=count_options,
+                    context_management=(
+                        None if context_management is None else context_management_with_edits(context_management.spec)
+                    ),
                 )
             except (PartnerVertexAIError, httpx.HTTPStatusError) as e:
                 status_code: Final = e.response.status_code
@@ -1565,12 +1569,15 @@ class VertexAITokenCounter(BaseTokenCounter):
                     original_response=result,
                 )
         else:
+            from litellm.llms.anthropic.messages.context_management.inference_view import (
+                InferenceRequestViewError,
+                count_context_edits,
+                inference_request_view,
+            )
             from litellm.llms.vertex_ai.count_tokens.context import (
                 VertexGeminiCountContextError,
-                count_context_edits,
                 count_context_error_response,
                 gemini_count_contents,
-                inference_request_view,
                 malformed_tool_result_text_count_error,
                 provider_error_response,
                 unconvertible_messages_error,
@@ -1584,7 +1591,7 @@ class VertexAITokenCounter(BaseTokenCounter):
                 messages_format,
                 count_context_edits(context_management, tools, count_tokens_params_request, model_to_use),
             )
-            if isinstance(request_view, VertexGeminiCountContextError):
+            if isinstance(request_view, InferenceRequestViewError):
                 return count_context_error_response(request_view, request_model, model_to_use)
             count_context: Final = vertex_gemini_count_context(
                 system=request_view.system, tools=tools, model=model_to_use

litellm/llms/vertex_ai/count_tokens/context.py

@@ -11,10 +11,11 @@ from litellm.litellm_core_utils.prompt_templates.factory import chat_completion_
 from litellm.types.llms.anthropic import AnthropicMessagesTextParam
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.llms.vertex_ai import ContentType
-from litellm.types.utils import CountContextManagement, TokenCountMessagesFormat, TokenCountResponse
+from litellm.types.utils import TokenCountMessagesFormat, TokenCountResponse
 
 if TYPE_CHECKING:
     from litellm.exceptions import APIError
+    from litellm.llms.anthropic.messages.context_management.inference_view import InferenceRequestViewError
 
 
 @dataclass(frozen=True, slots=True)
@@ -152,7 +153,7 @@ def malformed_tool_result_text_count_error(
 
 
 def count_context_error_response(
-    error: VertexGeminiCountContextError, request_model: str, model_used: str
+    error: "VertexGeminiCountContextError | InferenceRequestViewError", request_model: str, model_used: str
 ) -> TokenCountResponse:
     return TokenCountResponse(
         total_tokens=0,
@@ -178,130 +179,6 @@ def provider_error_response(error: "APIError", request_model: str, model_used: s
     )
 
 
-@dataclass(frozen=True, slots=True)
-class InferenceRequestView:
-    messages: Sequence[Mapping[str, object]]
-    system: object | None
-
-
-@dataclass(frozen=True, slots=True)
-class CountContextEdits:
-    spec: JsonValue
-    model: str
-    tools: tuple[Mapping[str, object], ...] | None
-    drop_invalid_edits: bool = False
-
-
-_CONTEXT_MANAGEMENT_SPEC: Final[TypeAdapter[Mapping[str, object] | tuple[Mapping[str, object], ...]]] = TypeAdapter(
-    Mapping[str, object] | tuple[Mapping[str, object], ...]
-)
-
-
-def count_context_edits(
-    context_management: CountContextManagement | None,
-    tools: Sequence[Mapping[str, object]] | None,
-    litellm_params: Mapping[str, object],
-    fallback_model: str,
-) -> CountContextEdits:
-    model: Final = litellm_params.get("model")
-    return CountContextEdits(
-        spec=None if context_management is None else context_management.spec,
-        model=model if isinstance(model, str) else fallback_model,
-        tools=None if tools is None else tuple(tools),
-        drop_invalid_edits=context_management is not None and context_management.drop_invalid_edits,
-    )
-
-
-def _typed_spec(spec: JsonValue) -> Mapping[str, object] | tuple[Mapping[str, object], ...] | None:
-    try:
-        return _CONTEXT_MANAGEMENT_SPEC.validate_python(spec)
-    except ValidationError:
-        return None
-
-
-def _typed_system(
-    system: object | None,
-) -> str | tuple[AnthropicMessagesTextParam, ...] | None | VertexGeminiCountContextError:
-    if system is None or isinstance(system, str):
-        return system
-    try:
-        return _SYSTEM_BLOCKS.validate_python(system)
-    except ValidationError:
-        return _INVALID_SYSTEM
-
-
-def _view_without_edits(
-    messages: Sequence[Mapping[str, object]],
-    system: object | None,
-    typed_system: str | Sequence[Mapping[str, object]] | None,
-) -> InferenceRequestView:
-    from litellm.llms.anthropic.messages.context_management.editors.compact import (
-        apply_client_compaction_block_history,
-    )
-
-    history: Final = apply_client_compaction_block_history(
-        messages=[dict(message) for message in messages],
-        system=typed_system
-        if typed_system is None or isinstance(typed_system, str)
-        else [dict(block) for block in typed_system],
-    )
-    if history is None:
-        return InferenceRequestView(messages=messages, system=system)
-    return InferenceRequestView(messages=history.messages, system=history.system)
-
-
-async def _anthropic_inference_view(
-    messages: Sequence[Mapping[str, object]], system: object | None, edits: CountContextEdits
-) -> InferenceRequestView | VertexGeminiCountContextError:
-    from litellm.llms.anthropic.messages.adapters.handler import prepare_context_managed_request
-    from litellm.llms.anthropic.messages.context_management import AnthropicContextManagementError
-    from litellm.llms.anthropic.messages.native.utils import normalize_native_messages
-
-    typed_system: Final = _typed_system(system)
-    if isinstance(typed_system, VertexGeminiCountContextError):
-        return typed_system
-    normalized: Final = normalize_native_messages(messages)
-    spec: Final = _typed_spec(edits.spec)
-    try:
-        prepared: Final = await prepare_context_managed_request(
-            model=edits.model,
-            messages=normalized,
-            tools=None if edits.tools is None else [dict(tool) for tool in edits.tools],
-            system=[dict(block) for block in typed_system] if isinstance(typed_system, tuple) else typed_system,
-            context_management_spec=(
-                None if spec is None else dict(spec) if isinstance(spec, Mapping) else [dict(edit) for edit in spec]
-            ),
-            litellm_metadata=None,
-            additional_drop_params=None,
-            llm_router=None,
-            count_only=True,
-            drop_invalid_edits=edits.drop_invalid_edits,
-        )
-    except AnthropicContextManagementError as error:
-        if error.status_code >= 500:
-            return _view_without_edits(normalized, system, typed_system)
-        return VertexGeminiCountContextError(error_message=error.message, status_code=error.status_code)
-    if prepared is None:
-        return InferenceRequestView(messages=normalized, system=system)
-    return InferenceRequestView(messages=prepared.messages, system=prepared.system)
-
-
-async def inference_request_view(
-    messages: Sequence[Mapping[str, object]],
-    system: object | None,
-    messages_format: TokenCountMessagesFormat,
-    edits: CountContextEdits,
-) -> InferenceRequestView | VertexGeminiCountContextError:
-    """Apply the history rewrites and context edits /v1/messages runs before it converts a body for Gemini."""
-    match messages_format:
-        case "openai":
-            return InferenceRequestView(messages=messages, system=system)
-        case "anthropic":
-            return await _anthropic_inference_view(messages, system, edits)
-        case _:
-            assert_never(messages_format)
-
-
 def _chat_messages(
     messages: Sequence[Mapping[str, object]],
     messages_format: TokenCountMessagesFormat,

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -31,9 +31,8 @@ from litellm.exceptions import UnsupportedParamsError
 from litellm.litellm_core_utils.core_helpers import as_str_mapping, drops_unsupported_params, normalize_drop_params
 from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
 from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
-from litellm.litellm_core_utils.prompt_templates.factory import (
-    encode_tool_call_id_with_signature,
-)
+from litellm.litellm_core_utils.model_param_helper import embeds_thought_signature_in_tool_call_id
+from litellm.litellm_core_utils.prompt_templates.factory import encode_tool_call_id_with_signature
 from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
@@ -438,6 +437,8 @@ _NAMED_UNKNOWN_VOICE_KEYS: Final = 10
 
 _NAMED_VOICE_KEY_CHARS: Final = 100
 
+_NAMED_AUDIO_FORMAT_CHARS: Final = 100
+
 
 def _gemini_speech_config_wrapper(value: Mapping[str, object]) -> Mapping[str, object] | None:
     camel_config: Final = as_str_mapping(value.get("speechConfig"))
@@ -474,6 +475,31 @@ def reject_unknown_gemini_tts_voice_keys(
     )
 
 
+def _named_audio_format(audio_format: object) -> str:
+    if isinstance(audio_format, str):
+        return repr(audio_format[:_NAMED_AUDIO_FORMAT_CHARS])
+    return repr(audio_format)[:_NAMED_AUDIO_FORMAT_CHARS]
+
+
+def unsupported_gemini_tts_audio_format(
+    model: str, audio: Mapping[str, object], drop_params: bool, llm_provider: str | None
+) -> UnsupportedParamsError | None:
+    nested_config: Final = _gemini_speech_config_wrapper(audio) or {}
+    audio_format: Final = next(
+        (item for item in (audio.get("format"), nested_config.get("format")) if item not in (None, "pcm16")), None
+    )
+    if audio_format is None or drops_unsupported_params(drop_params):
+        return None
+    return UnsupportedParamsError(
+        message=(
+            f"{model} does not take the audio format {_named_audio_format(audio_format)}. Set format "
+            "to 'pcm16' or leave it out, or set drop_params to drop it."
+        ),
+        model=model,
+        llm_provider=llm_provider,
+    )
+
+
 def normalize_gemini_speech_config(
     value: Mapping[str, object], drop_unknown_fields: bool = False
 ) -> dict[str, object]:  # mutable-ok: provider request serialization requires a concrete dict
@@ -1556,15 +1582,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             languageCode: "en-US",
         }
         """
-        # Validate audio format - Gemini TTS only supports pcm16
-        audio_format: Final = value.get("format")
-        if audio_format is not None and audio_format != "pcm16":
-            raise ValueError(
-                f"Unsupported audio format for Gemini TTS models: {audio_format}. "
-                f"Gemini TTS models only support 'pcm16' format as they return audio data in L16 PCM format. "
-                f"Please set audio format to 'pcm16'."
-            )
-
+        audio_format_error: Final = unsupported_gemini_tts_audio_format(
+            "Gemini TTS", value, drop_unknown_fields, self.custom_llm_provider
+        )
+        if audio_format_error is not None:
+            raise audio_format_error
         camel_speech_config: Final = value.get("speechConfig")
         snake_speech_config: Final = value.get("speech_config")
         speech_config_value: Final = (
@@ -1602,6 +1624,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
     def _map_chat_audio_params(
         self, model: str, value: Mapping[str, object], drop_params: bool
     ) -> dict[str, object]:  # mutable-ok: the public forwarding interface returns a concrete request dictionary
+        audio_format_error: Final = unsupported_gemini_tts_audio_format(
+            model, value, drop_params, self.custom_llm_provider
+        )
+        if audio_format_error is not None:
+            raise audio_format_error
         if _gemini_speech_config_wrapper(value) is not None:
             reject_unknown_gemini_tts_voice_keys(model, value, drop_params, self.custom_llm_provider)
         return self._map_audio_params(value, drop_unknown_fields=drops_unsupported_params(drop_params))
@@ -2091,6 +2118,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         parts: list[HttpxPartType],
         cumulative_tool_call_idx: int,
         is_function_call: bool | None,
+        thought_signature_in_tool_call_id: bool = True,
     ) -> tuple[
         ChatCompletionToolCallFunctionChunk | None,
         list[ChatCompletionToolCallChunk] | None,
@@ -2129,9 +2157,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                     # back on the matching `functionResponse`.
                     if gemini_call_id:
                         _tool_response_chunk["id"] = gemini_call_id
-                    # Embed thought signature in ID for OpenAI client compatibility
                     if thought_signature:
                         _tool_response_chunk["provider_specific_fields"] = {"thought_signature": thought_signature}
+                    if thought_signature and thought_signature_in_tool_call_id:
                         _tool_response_chunk["id"] = encode_tool_call_id_with_signature(
                             _tool_response_chunk["id"] or "", thought_signature
                         )
@@ -2793,6 +2821,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         model_response: Union[ModelResponse, "ModelResponseStream"],
         standard_optional_params: dict,
         cumulative_tool_call_index: int = 0,
+        thought_signature_in_tool_call_id: bool = True,
     ) -> tuple[list[dict], list[dict], list, list, int]:
         """
         Helper method to process candidates and extract metadata
@@ -2895,6 +2924,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                     parts=candidate["content"]["parts"],
                     cumulative_tool_call_idx=cumulative_tool_call_index,
                     is_function_call=is_function_call(standard_optional_params),
+                    thought_signature_in_tool_call_id=thought_signature_in_tool_call_id,
                 )
 
             if "logprobsResult" in candidate:
@@ -2978,6 +3008,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         model_response: Union[ModelResponse, "ModelResponseStream"],
         standard_optional_params: dict[str, object],  # mutable-ok: mirrors override contract
         cumulative_tool_call_index: int = 0,
+        thought_signature_in_tool_call_id: bool = True,
     ) -> tuple[  # mutable-ok: mirrors override contract
         list[dict[str, object]],
         list[dict[str, object]],
@@ -2986,7 +3017,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         int,
     ]:
         return cls._process_candidates(
-            _candidates, model_response, standard_optional_params, cumulative_tool_call_index
+            _candidates,
+            model_response,
+            standard_optional_params,
+            cumulative_tool_call_index,
+            thought_signature_in_tool_call_id,
         )
 
     def transform_response(
@@ -3069,7 +3104,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                     safety_ratings,
                     citation_metadata,
                     _,  # cumulative_tool_call_index not needed in non-streaming
-                ) = VertexGeminiConfig._process_candidates(_candidates, model_response, logging_obj.optional_params)
+                ) = VertexGeminiConfig._process_candidates(
+                    _candidates,
+                    model_response,
+                    logging_obj.optional_params,
+                    thought_signature_in_tool_call_id=embeds_thought_signature_in_tool_call_id(
+                        logging_obj.litellm_params  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # Logging.litellm_params is untyped
+                    ),
+                )
 
             usage: Final = VertexGeminiConfig._calculate_usage(
                 completion_response=completion_response,
@@ -3905,6 +3947,9 @@ class ModelResponseIterator:
         self.logging_obj = logging_obj
         self.response_headers = response_headers or {}
         self.is_function_call = check_is_function_call(logging_obj)
+        self.thought_signature_in_tool_call_id: Final = embeds_thought_signature_in_tool_call_id(
+            logging_obj.litellm_params  # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType]  # Logging.litellm_params is untyped
+        )
         self.cumulative_tool_call_index: int = 0
         self.has_seen_tool_calls: bool = False
         self._resume = resume
@@ -3969,6 +4014,7 @@ class ModelResponseIterator:
             model_response,
             self.logging_obj.optional_params,
             cumulative_tool_call_index=self.cumulative_tool_call_index,
+            thought_signature_in_tool_call_id=self.thought_signature_in_tool_call_id,
         )
 
         # Track whether tool_calls have been seen across streaming chunks.

litellm/llms/vertex_ai/vertex_ai_partner_models/count_tokens/handler.py

@@ -9,12 +9,14 @@ their respective publisher-specific count-tokens endpoints.
 from collections.abc import Mapping, Sequence
 from typing import Any, Final
 
-from pydantic import TypeAdapter
+from pydantic import JsonValue, TypeAdapter
 
+from litellm.anthropic_beta_headers_manager import aload_beta_headers_config, filter_and_transform_beta_headers
 from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.anthropic.common_utils import (
     OBJECT_LIST_ADAPTER,
+    context_management_betas,
     count_tokens_feature_betas,
     is_claude_model_target,
     tools_without_custom_field,
@@ -31,6 +33,7 @@ _COUNT_TOKENS_PUBLISHERS: Final = (
 
 
 _REQUEST_MESSAGES: Final = TypeAdapter(Sequence[object])
+_CONTEXT_MANAGEMENT: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
 
 
 def count_tokens_request_body(
@@ -55,8 +58,14 @@ def count_tokens_publisher(model: str) -> str | None:
 
 def count_tokens_headers(access_token: str, request_data: Mapping[str, object]) -> Mapping[str, str]:
     authorization: Final = {"Authorization": f"Bearer {access_token}"}
-    betas: Final = count_tokens_feature_betas(
-        "vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
+    betas: Final = (
+        *count_tokens_feature_betas(
+            "vertex_ai", request_data, _REQUEST_MESSAGES.validate_python(request_data["messages"])
+        ),
+        *filter_and_transform_beta_headers(
+            list(context_management_betas(_CONTEXT_MANAGEMENT.validate_python(request_data.get("context_management")))),
+            "vertex_ai",
+        ),
     )
     return {**authorization, "anthropic-beta": ",".join(betas)} if betas else authorization
 
@@ -193,6 +202,8 @@ class VertexAIPartnerModelsTokenCounter(VertexBase):
             api_base=litellm_params.get("api_base"),
         )
 
+        if request_data.get("context_management") is not None:
+            await aload_beta_headers_config()
         headers: Final = dict(count_tokens_headers(access_token, request_data))
 
         # Get async HTTP client

litellm/llms/vertex_ai/vertex_ai_partner_models/main.py

@@ -268,6 +268,7 @@ class VertexAIPartnerModels(VertexBase):
         system: object | None = None,
         tools: list[dict[str, object]] | None = None,
         count_options: Mapping[str, object] = MappingProxyType({}),
+        context_management: object = None,
     ):
         """
         Count tokens for Vertex AI partner models (Anthropic Claude, Mistral, etc.)
@@ -302,7 +303,15 @@ class VertexAIPartnerModels(VertexBase):
                 "model": model,
                 "messages": messages,
                 **MappingProxyType(
-                    {key: value for key, value in (("system", system), ("tools", tools)) if value is not None}
+                    {
+                        key: value
+                        for key, value in (
+                            ("system", system),
+                            ("tools", tools),
+                            ("context_management", context_management),
+                        )
+                        if value is not None
+                    }
                 ),
                 **count_options,
             }

litellm/main.py

@@ -23,6 +23,7 @@ from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Mappin
 from concurrent import futures
 from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 from copy import deepcopy
+from dataclasses import dataclass
 from functools import partial
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Literal, Optional, Protocol, Union, cast, get_args
@@ -82,8 +83,8 @@ from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.get_litellm_params import (
     AWS_CREDENTIAL_KWARGS_KEYS,
     OAUTH_TOKEN_EXCHANGE_KWARGS_KEYS,
-    OPTIONAL_KWARGS_KEYS,
     PROVIDER_AFFINITY_HEADER_KWARG_KEY,
+    SUPPLEMENTAL_KWARGS_KEYS,
     InvalidControlOption,
     parse_control_options,
     with_control_options,
@@ -157,7 +158,7 @@ from litellm.types.completion import (
 )
 from litellm.types.litellm_params import ControlOptions, RetryStrategy
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.router import GenericLiteLLMParams
+from litellm.types.router import GenericLiteLLMParams, validate_litellm_params_strictly
 from litellm.types.utils import (
     AgenticLoopParams,
     BaseUrlRoutingHint,
@@ -399,6 +400,49 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro
             return assert_never(control)
 
 
+@dataclass(frozen=True, slots=True)
+class _SupplementalProviderParams:
+    params: GenericLiteLLMParams
+    dropped_keys: frozenset[str] = frozenset()
+
+
+@dataclass(frozen=True, slots=True)
+class _InvalidSupplementalParams:
+    message: str
+    param: str
+
+
+def _parse_supplemental_provider_params(
+    kwargs: Mapping[str, object],
+) -> _SupplementalProviderParams | _InvalidSupplementalParams:
+    supplemental: Final = {key: kwargs[key] for key in SUPPLEMENTAL_KWARGS_KEYS if key in kwargs}
+    try:
+        return _SupplementalProviderParams(params=validate_litellm_params_strictly(GenericLiteLLMParams, supplemental))
+    except ValidationError as error:
+        invalid: Final = tuple(detail for detail in error.errors() if detail["loc"])
+        invalid_keys: Final = frozenset(str(detail["loc"][0]) for detail in invalid)
+        if litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True:
+            return _SupplementalProviderParams(
+                params=GenericLiteLLMParams.model_validate(
+                    {key: value for key, value in supplemental.items() if key not in invalid_keys}
+                ),
+                dropped_keys=invalid_keys,
+            )
+        return _InvalidSupplementalParams(
+            message="; ".join(f"{detail['loc'][0]}: {detail['msg']}" for detail in invalid),
+            param=", ".join(sorted(invalid_keys)),
+        )
+
+
+def _supplemental_provider_params(kwargs: Mapping[str, object], model: str) -> _SupplementalProviderParams:
+    parsed: Final = _parse_supplemental_provider_params(kwargs)
+    if isinstance(parsed, _InvalidSupplementalParams):
+        raise litellm.BadRequestError(
+            message=parsed.message, model=model, llm_provider=None, body={"param": parsed.param}
+        )
+    return parsed
+
+
 PROVIDERS_WITHOUT_DOCUMENT_INPUT: Final = frozenset({"ollama", "ollama_chat"})
 PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT: Final = frozenset({"azure_ai"})
 
@@ -830,21 +874,17 @@ async def acompletion(
         "enable_json_schema_validation": enable_json_schema_validation,
     }
     if custom_llm_provider is None:
-        _supplemental: Final = cast(  # cast-ok: kwargs values are request-scoped objects
-            "dict[str, object]",
-            {
-                **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
-                "extra_headers": extra_headers,
-            },
-        )
+        request_kwargs: Final = cast("Mapping[str, object]", kwargs)  # cast-ok: untyped request kwargs
         _, custom_llm_provider, _, _ = get_llm_provider(
             model=model,
             custom_llm_provider=custom_llm_provider,
             api_base=cast(  # cast-ok: api_base arrives as a str in the request kwargs
-                "str | None", kwargs.get("api_base")
+                "str | None", request_kwargs.get("api_base")
             )
             or base_url,
-            litellm_params=(GenericLiteLLMParams.model_validate(_supplemental) if _supplemental else None),
+            litellm_params=_supplemental_provider_params(
+                _REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model
+            ).params,
         )
 
     if fallbacks is not None:
@@ -1197,6 +1237,7 @@ def mock_completion(
                     custom_llm_provider="openai",
                     logging_obj=logging,
                     count_prompt_tokens=count_prompt_tokens,
+                    is_mock=True,
                 )
             return CustomStreamWrapper(
                 completion_stream=mock_completion_streaming_obj(
@@ -1206,6 +1247,7 @@ def mock_completion(
                 custom_llm_provider="openai",
                 logging_obj=logging,
                 count_prompt_tokens=count_prompt_tokens,
+                is_mock=True,
             )
         if isinstance(mock_response, litellm.MockException):
             raise mock_response
@@ -4382,6 +4424,7 @@ _ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str])
 _OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
 _OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str])
 _MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
+_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 _PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
 _BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint)
 _OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object)
@@ -5625,6 +5668,11 @@ def completion(
 
     ######### unpacking kwargs #####################
     args: Final = _locals_snapshot(locals())
+    supplemental_params: Final = _supplemental_provider_params(
+        _REQUEST_KWARGS_ADAPTER.validate_python({**kwargs, "extra_headers": extra_headers}), model
+    )
+    for dropped_key in supplemental_params.dropped_keys:
+        kwargs.pop(dropped_key)
 
     # Set by the responses->completion fallback so completion() does not bridge
     # back to the Responses API: that round-trip mutually recurses forever for a
@@ -5850,21 +5898,14 @@ def completion(
         if deployment_id is not None:  # azure llms
             model = deployment_id
             custom_llm_provider = "azure"
-        _supplemental_provider_params: Final = {
-            **{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
-            "extra_headers": extra_headers,
-        }
+        routing_kwargs: Final = cast("Mapping[str, object]", kwargs)  # cast-ok: kwargs is the untyped request dict
         requested_model: Final = model
         model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
             model=model,
             custom_llm_provider=custom_llm_provider,
             api_base=api_base,
             api_key=api_key,
-            litellm_params=(
-                GenericLiteLLMParams.model_validate(_supplemental_provider_params)
-                if _supplemental_provider_params
-                else None
-            ),
+            litellm_params=supplemental_params.params,
         )
         _record_agentic_loop_params(
             logging_obj=logging,
@@ -5884,7 +5925,6 @@ def completion(
             api_base=api_base,
             deployment_model_info=deployment_model_info,
         )
-        routing_kwargs: Final = cast("Mapping[str, object]", kwargs)  # cast-ok: kwargs is the untyped request dict
         drop_unsupported_reasoning: Final = (
             litellm.drop_params is True or normalize_drop_params(routing_kwargs.get("drop_params")) is True
         )
@@ -6220,6 +6260,7 @@ def completion(
                     "github_copilot_user_session",
                     PROVIDER_AFFINITY_HEADER_KWARG_KEY,
                     "fireworks_forward_user_id",
+                    "thought_signature_in_tool_call_id",
                     "responses_api_version",
                     "allowed_openai_params",
                 )

litellm/messages/dispatch.py

@@ -75,9 +75,7 @@ def _public_request(
     return native_call(legacy, args, _native_kwargs(fields, kwargs))
 
 
-def _resolved(request: NativeCall) -> tuple[str, str | None]:
-    model: Final = str(request.resolved["model"])
-    provider: Final = optional_str(request.resolved.get("custom_llm_provider"))
+def _resolved_model(model: str, provider: str | None) -> tuple[str, str | None]:
     try:
         resolved_model, resolved_provider, _, _ = get_llm_provider(model, provider)
     except BadRequestError:
@@ -85,21 +83,43 @@ def _resolved(request: NativeCall) -> tuple[str, str | None]:
     return resolved_model, resolved_provider
 
 
-def _target(request: NativeCall, model: str, provider: str | None) -> Target:
-    requested_model: Final = str(request.resolved["model"])
-    model_info: Final = request.resolved.get("model_info")
-    api_base: Final = request.resolved.get("api_base")
-    native_config: Final = main._native_messages_config(  # pyright: ignore[reportPrivateUsage]  # the handler's own passthrough lookup
-        model, provider, model_info, api_base if isinstance(api_base, str) else None
-    )
+def _resolved(request: NativeCall) -> tuple[str, str | None]:
+    return _resolved_model(str(request.resolved["model"]), optional_str(request.resolved.get("custom_llm_provider")))
+
+
+def _route_target(
+    requested_model: str,
+    model: str,
+    provider: str | None,
+    model_info: object,
+    api_base: str | None,
+    compacting: bool,
+) -> Target:
+    native_config: Final = main._native_messages_config(model, provider, model_info, api_base)  # pyright: ignore[reportPrivateUsage]  # the handler's own passthrough lookup
     if native_config is not None:
         return Target.MESSAGES
-    compacting: Final = request.resolved.get("compaction") is not None
     if not compacting and main._should_route_to_responses_api(provider, requested_model, model, model_info):  # pyright: ignore[reportPrivateUsage]  # the handler's own translation choice
         return Target.RESPONSES
     return Target.CHAT_COMPLETIONS
 
 
+def _target(request: NativeCall, model: str, provider: str | None) -> Target:
+    api_base: Final = request.resolved.get("api_base")
+    return _route_target(
+        str(request.resolved["model"]),
+        model,
+        provider,
+        request.resolved.get("model_info"),
+        api_base if isinstance(api_base, str) else None,
+        compacting=request.resolved.get("compaction") is not None,
+    )
+
+
+def messages_target(model: str, custom_llm_provider: str | None, model_info: object, api_base: str | None) -> Target:
+    resolved_model, resolved_provider = _resolved_model(model, custom_llm_provider)
+    return _route_target(model, resolved_model, resolved_provider, model_info, api_base, compacting=False)
+
+
 _FEATURES: Final[detectors.Detectors] = MappingProxyType(
     {
         **detectors.COMMON,

litellm/model_prices_and_context_window_backup.json

@@ -15555,18 +15555,20 @@
         "input_cost_per_token": 2.65e-06,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8000,
-        "max_output_tokens": 2048,
+        "max_output_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 3.5e-06,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-70b-instruct.md",
         "supports_pdf_input": true
     },
     "bedrock/us-gov-west-1/meta.llama3-8b-instruct-v1:0": {
         "input_cost_per_token": 3e-07,
         "litellm_provider": "bedrock",
         "max_input_tokens": 8000,
-        "max_output_tokens": 2048,
+        "max_output_tokens": 8192,
         "mode": "chat",
         "output_cost_per_token": 6e-07,
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-meta-llama-3-8b-instruct.md",
         "supports_pdf_input": true
     },
     "bedrock/us-west-1/meta.llama3-70b-instruct-v1:0": {
@@ -48955,6 +48957,7 @@
     },
     "together_ai/Qwen/Qwen3.8-2.4T-A95B": {
         "cache_read_input_token_cost": 2.5e-07,
+        "deprecation_date": "2026-11-04",
         "input_cost_per_token": 2e-06,
         "litellm_provider": "together_ai",
         "max_input_tokens": 1010000,
@@ -53954,23 +53957,23 @@
         "supports_vision": true
     },
     "wandb/meta-llama/Llama-3.1-8B-Instruct": {
-        "max_input_tokens": 131000,
+        "max_input_tokens": 128000,
         "max_output_tokens": 128000,
         "input_cost_per_token": 2.2e-07,
         "output_cost_per_token": 2.2e-07,
         "litellm_provider": "wandb",
         "mode": "chat",
-        "source": "https://wandb.ai/site/pricing/tokens/"
+        "source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference"
     },
     "wandb/deepseek-ai/DeepSeek-V3.1": {
         "supports_reasoning": true,
-        "max_input_tokens": 161000,
+        "max_input_tokens": 128000,
         "max_output_tokens": 128000,
         "input_cost_per_token": 5.5e-07,
         "output_cost_per_token": 1.65e-06,
         "litellm_provider": "wandb",
         "mode": "chat",
-        "source": "https://wandb.ai/site/pricing/tokens/"
+        "source": "https://www.coreweave.com/products/coreweave-forge/serverless-inference"
     },
     "wandb/meta-llama/Llama-3.3-70B-Instruct": {
         "max_input_tokens": 128000,
@@ -59872,6 +59875,7 @@
         "max_input_tokens": 922000,
         "max_output_tokens": 128000,
         "mode": "responses",
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-cyber.html",
         "use_openai_responses_path": true,
         "supported_endpoints": [
@@ -59893,7 +59897,8 @@
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_xhigh_reasoning_effort": true,
-        "supports_vision": true
+        "supports_vision": true,
+        "supports_web_search": true
     },
     "bedrock_mantle/openai.gpt-daybreak-blue-5.6-sol": {
         "input_cost_per_token": 4.4e-06,
@@ -59908,6 +59913,7 @@
         "max_input_tokens": 1050000,
         "max_output_tokens": 128000,
         "mode": "responses",
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "use_openai_responses_path": true,
         "supported_endpoints": [
             "/v1/responses"
@@ -59927,7 +59933,8 @@
         "supports_tool_choice": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-daybreak-blue-56-sol.html"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-daybreak-blue-56-sol.html",
+        "supports_web_search": true
     },
     "bedrock_mantle/openai.gpt-5.6-luna": {
         "input_cost_per_token": 2.2e-07,
@@ -60333,6 +60340,7 @@
         "mode": "responses",
         "output_cost_per_token_above_272k_tokens_ultrafast": 0.000495,
         "output_cost_per_token_ultrafast": 0.00033,
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "use_openai_responses_path": true,
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -60356,7 +60364,8 @@
         "supports_tool_choice": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-astra.html",
+        "supports_web_search": true
     },
     "bedrock_mantle/openai.gpt-6-sol": {
         "input_cost_per_token": 2.2e-06,
@@ -60371,6 +60380,7 @@
         "max_input_tokens": 922000,
         "max_output_tokens": 128000,
         "mode": "responses",
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "use_openai_responses_path": true,
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -60394,7 +60404,8 @@
         "supports_tool_choice": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html",
+        "supports_web_search": true
     },
     "bedrock_mantle/openai.gpt-6-luna": {
         "input_cost_per_token": 1.1e-07,
@@ -60409,6 +60420,7 @@
         "max_input_tokens": 922000,
         "max_output_tokens": 128000,
         "mode": "responses",
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "use_openai_responses_path": true,
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -60432,7 +60444,8 @@
         "supports_tool_choice": true,
         "supports_xhigh_reasoning_effort": true,
         "supports_vision": true,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html"
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-cards-openai.html",
+        "supports_web_search": true
     },
     "us.openai.gpt-6-astra": {
         "supports_bedrock_runtime_chat_completions_response_format": true,
@@ -68630,7 +68643,9 @@
         "cache_read_input_token_cost_above_272k_tokens": 5.4e-07,
         "output_cost_per_token": 1.62e-05,
         "output_cost_per_token_above_272k_tokens": 2.43e-05,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html"
+        "search_context_cost_per_query": {"search_context_size_high":0.0144,"search_context_size_low":0.0144,"search_context_size_medium":0.0144},
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-terra.html",
+        "supports_web_search": true
     },
     "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-luna": {
         "litellm_provider": "bedrock_mantle",
@@ -68665,7 +68680,9 @@
         "cache_read_input_token_cost_above_272k_tokens": 5.4e-08,
         "output_cost_per_token": 1.62e-06,
         "output_cost_per_token_above_272k_tokens": 2.43e-06,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html"
+        "search_context_cost_per_query": {"search_context_size_high":0.0144,"search_context_size_low":0.0144,"search_context_size_medium":0.0144},
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-56-luna.html",
+        "supports_web_search": true
     },
     "bedrock_mantle/us-gov-west-1/openai.gpt-5.4": {
         "litellm_provider": "bedrock_mantle",
@@ -68694,7 +68711,9 @@
         "input_cost_per_token": 3.375e-06,
         "cache_read_input_token_cost": 3.375e-07,
         "output_cost_per_token": 2.025e-05,
-        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html"
+        "search_context_cost_per_query": {"search_context_size_high":0.0144,"search_context_size_low":0.0144,"search_context_size_medium":0.0144},
+        "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-54.html",
+        "supports_web_search": true
     },
     "bedrock_mantle/us-gov-west-1/xai.grok-4.3": {
         "use_openai_responses_path": true,
@@ -75309,6 +75328,7 @@
         "output_cost_per_token_above_272k_tokens": 1.65e-05,
         "output_cost_per_token_above_272k_tokens_ultrafast": 9.9e-05,
         "output_cost_per_token_ultrafast": 6.6e-05,
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-sol.html",
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -75331,6 +75351,7 @@
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true,
+        "supports_web_search": true,
         "supports_xhigh_reasoning_effort": true,
         "use_openai_responses_path": true
     },
@@ -79019,6 +79040,7 @@
         "mode": "responses",
         "output_cost_per_token": 5.5e-06,
         "output_cost_per_token_above_272k_tokens": 8.25e-06,
+        "search_context_cost_per_query": {"search_context_size_high":0.012,"search_context_size_low":0.012,"search_context_size_medium":0.012},
         "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-6-1-terra.html",
         "supported_endpoints": [
             "/v1/chat/completions",
@@ -79041,6 +79063,7 @@
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true,
+        "supports_web_search": true,
         "supports_xhigh_reasoning_effort": true,
         "use_openai_responses_path": true
     },
@@ -81372,5 +81395,77 @@
         "supports_response_schema": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true
+    },
+    "gpt-6-astra-law": {
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "openai",
+        "max_input_tokens": 922000,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "reasoning_contexts": [
+            "auto",
+            "current_turn",
+            "all_turns"
+        ],
+        "reasoning_modes": [
+            "standard",
+            "pro"
+        ],
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
+        "source": "https://developers.openai.com/api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": false,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_cache_breakpoint": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_tool_search": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "chatgpt/gpt-6-astra-law": {
+        "litellm_provider": "chatgpt",
+        "max_input_tokens": 922000,
+        "max_output_tokens": 128000,
+        "mode": "responses",
+        "source": "https://developers.openai.com/api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_none_reasoning_effort": false,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
     }
 }

litellm/proxy/_lazy_features.py

@@ -105,10 +105,10 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
     LazyFeature(
         name="a2a",
         module_path="litellm.proxy.agent_endpoints.a2a_endpoints",
-        # ``/v1/a2a/{agent_id}/message/send`` is caught via the suffix so the
+        # ``/v1/a2a/{agent_id}/message/send`` and ``/message/stream`` are caught via suffixes so the
         # ``/v1/a2a`` prefix doesn't subsume the discover prefix below.
         path_prefixes=("/a2a",),
-        path_suffixes=("/message/send",),
+        path_suffixes=("/message/send", "/message/stream"),
     ),
     LazyFeature(
         name="a2a_registration",

litellm/proxy/_lazy_openapi_snapshot.json

@@ -54,6 +54,57 @@
           ],
           "title": "ValidationError",
           "type": "object"
+        },
+        "_A2AJsonRpcResponse": {
+          "additionalProperties": true,
+          "properties": {
+            "error": {
+              "anyOf": [
+                {
+                  "additionalProperties": true,
+                  "type": "object"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Error"
+            },
+            "id": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "integer"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Id"
+            },
+            "jsonrpc": {
+              "const": "2.0",
+              "title": "Jsonrpc",
+              "type": "string"
+            },
+            "result": {
+              "anyOf": [
+                {},
+                {
+                  "type": "null"
+                }
+              ],
+              "title": "Result"
+            }
+          },
+          "required": [
+            "jsonrpc",
+            "id"
+          ],
+          "title": "_A2AJsonRpcResponse",
+          "type": "object"
         }
       }
     },
@@ -242,6 +293,119 @@
           ]
         }
       },
+      "/a2a/{agent_id}/message/stream": {
+        "post": {
+          "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response",
+          "operationId": "invoke_agent_a2a_a2a__agent_id__message_stream_post",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "agent_id",
+              "required": true,
+              "schema": {
+                "title": "Agent Id",
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "content": {
+              "application/json": {
+                "example": {
+                  "id": "stream-1",
+                  "jsonrpc": "2.0",
+                  "method": "message/stream",
+                  "params": {
+                    "message": {
+                      "kind": "message",
+                      "messageId": "message-1",
+                      "parts": [
+                        {
+                          "kind": "text",
+                          "text": "Hello"
+                        }
+                      ],
+                      "role": "user"
+                    }
+                  }
+                },
+                "schema": {
+                  "additionalProperties": true,
+                  "properties": {
+                    "id": {
+                      "anyOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "integer"
+                        },
+                        {
+                          "type": "null"
+                        }
+                      ],
+                      "title": "Id"
+                    },
+                    "jsonrpc": {
+                      "const": "2.0",
+                      "title": "Jsonrpc",
+                      "type": "string"
+                    },
+                    "method": {
+                      "title": "Method",
+                      "type": "string"
+                    },
+                    "params": {
+                      "additionalProperties": true,
+                      "title": "Params",
+                      "type": "object"
+                    }
+                  },
+                  "required": [
+                    "jsonrpc",
+                    "method",
+                    "params"
+                  ],
+                  "title": "_A2AJsonRpcRequest",
+                  "type": "object"
+                }
+              }
+            },
+            "required": true
+          },
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/_A2AJsonRpcResponse"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Invoke Agent A2A",
+          "tags": [
+            "a2a"
+          ]
+        }
+      },
       "/v1/a2a/{agent_id}/message/send": {
         "post": {
           "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response",
@@ -287,6 +451,119 @@
             "a2a"
           ]
         }
+      },
+      "/v1/a2a/{agent_id}/message/stream": {
+        "post": {
+          "description": "Invoke an agent using the A2A protocol (JSON-RPC 2.0).\n\nSupported methods:\n- message/send: Send a message and get a response\n- message/stream: Send a message and stream the response",
+          "operationId": "invoke_agent_a2a_v1_a2a__agent_id__message_stream_post",
+          "parameters": [
+            {
+              "in": "path",
+              "name": "agent_id",
+              "required": true,
+              "schema": {
+                "title": "Agent Id",
+                "type": "string"
+              }
+            }
+          ],
+          "requestBody": {
+            "content": {
+              "application/json": {
+                "example": {
+                  "id": "stream-1",
+                  "jsonrpc": "2.0",
+                  "method": "message/stream",
+                  "params": {
+                    "message": {
+                      "kind": "message",
+                      "messageId": "message-1",
+                      "parts": [
+                        {
+                          "kind": "text",
+                          "text": "Hello"
+                        }
+                      ],
+                      "role": "user"
+                    }
+                  }
+                },
+                "schema": {
+                  "additionalProperties": true,
+                  "properties": {
+                    "id": {
+                      "anyOf": [
+                        {
+                          "type": "string"
+                        },
+                        {
+                          "type": "integer"
+                        },
+                        {
+                          "type": "null"
+                        }
+                      ],
+                      "title": "Id"
+                    },
+                    "jsonrpc": {
+                      "const": "2.0",
+                      "title": "Jsonrpc",
+                      "type": "string"
+                    },
+                    "method": {
+                      "title": "Method",
+                      "type": "string"
+                    },
+                    "params": {
+                      "additionalProperties": true,
+                      "title": "Params",
+                      "type": "object"
+                    }
+                  },
+                  "required": [
+                    "jsonrpc",
+                    "method",
+                    "params"
+                  ],
+                  "title": "_A2AJsonRpcRequest",
+                  "type": "object"
+                }
+              }
+            },
+            "required": true
+          },
+          "responses": {
+            "200": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/_A2AJsonRpcResponse"
+                  }
+                }
+              },
+              "description": "Successful Response"
+            },
+            "422": {
+              "content": {
+                "application/json": {
+                  "schema": {
+                    "$ref": "#/components/schemas/HTTPValidationError"
+                  }
+                }
+              },
+              "description": "Validation Error"
+            }
+          },
+          "security": [
+            {
+              "APIKeyHeader": []
+            }
+          ],
+          "summary": "Invoke Agent A2A",
+          "tags": [
+            "a2a"
+          ]
+        }
       }
     }
   },
@@ -15326,6 +15603,10 @@
               ],
               "title": "Metadata"
             },
+            "response_format": {
+              "$ref": "#/components/schemas/GuardrailResponseFormat",
+              "default": "standard"
+            },
             "text": {
               "title": "Text",
               "type": "string"
@@ -16203,6 +16484,14 @@
           "title": "BedrockChecksSensitiveInformationModel",
           "type": "object"
         },
+        "BedrockGuardrailOutputScope": {
+          "enum": [
+            "INTERVENTIONS",
+            "FULL"
+          ],
+          "title": "BedrockGuardrailOutputScope",
+          "type": "string"
+        },
         "BlockedWord": {
           "description": "Represents a blocked word with its action and optional description",
           "properties": {
@@ -16746,6 +17035,14 @@
           "title": "GuardrailInfoResponse",
           "type": "object"
         },
+        "GuardrailResponseFormat": {
+          "enum": [
+            "standard",
+            "native"
+          ],
+          "title": "GuardrailResponseFormat",
+          "type": "string"
+        },
         "GuardrailSubmissionItem": {
           "properties": {
             "created_at": {
@@ -16998,6 +17295,7 @@
           "title": "HTTPValidationError",
           "type": "object"
         },
+        "JsonValue": {},
         "LakeraCategoryThresholds": {
           "additionalProperties": true,
           "properties": {
@@ -17924,6 +18222,18 @@
               "description": "Whether to fail the request if the guardrail encounters an error. Implemented by guardrail='model_armor', 'generic_guardrail_api' and 'crowdstrike_aidr'. True (default) raises the error. False logs a critical error and lets the request proceed, so only a valid guardrail response can block or modify it.",
               "title": "Fail On Error"
             },
+            "file_checkpoint_id": {
+              "anyOf": [
+                {
+                  "type": "string"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "File-checkpoint ID for the Ovalix Tracker service (falls back to pre/post).",
+              "title": "File Checkpoint Id"
+            },
             "file_guardrail_timeout": {
               "anyOf": [
                 {
@@ -18426,6 +18736,17 @@
               ],
               "description": "Optional parameters for the guardrail"
             },
+            "outputScope": {
+              "anyOf": [
+                {
+                  "$ref": "#/components/schemas/BedrockGuardrailOutputScope"
+                },
+                {
+                  "type": "null"
+                }
+              ],
+              "description": "ApplyGuardrail response scope. FULL includes detected and non-detected findings. Omit to retain Bedrock's default of returning detected findings only."
+            },
             "output_parse_pii": {
               "anyOf": [
                 {
@@ -20308,7 +20629,24 @@
               "content": {
                 "application/json": {
                   "schema": {
-                    "$ref": "#/components/schemas/ApplyGuardrailResponse"
+                    "anyOf": [
+                      {
+                        "$ref": "#/components/schemas/ApplyGuardrailResponse"
+                      },
+                      {
+                        "additionalProperties": {
+                          "$ref": "#/components/schemas/JsonValue"
+                        },
+                        "type": "object"
+                      },
+                      {
+                        "items": {
+                          "$ref": "#/components/schemas/JsonValue"
+                        },
+                        "type": "array"
+                      }
+                    ],
+                    "title": "Response Apply Guardrail Apply Guardrail Post"
                   }
                 }
               },
@@ -20400,7 +20738,24 @@
               "content": {
                 "application/json": {
                   "schema": {
-                    "$ref": "#/components/schemas/ApplyGuardrailResponse"
+                    "anyOf": [
+                      {
+                        "$ref": "#/components/schemas/ApplyGuardrailResponse"
+                      },
+                      {
+                        "additionalProperties": {
+                          "$ref": "#/components/schemas/JsonValue"
+                        },
+                        "type": "object"
+                      },
+                      {
+                        "items": {
+                          "$ref": "#/components/schemas/JsonValue"
+                        },
+                        "type": "array"
+                      }
+                    ],
+                    "title": "Response Apply Guardrail Guardrails Apply Guardrail Post"
                   }
                 }
               },
@@ -24268,7 +24623,14 @@
           "properties": {
             "content": {
               "items": {
-                "$ref": "#/components/schemas/OutputText"
+                "anyOf": [
+                  {
+                    "$ref": "#/components/schemas/OutputText"
+                  },
+                  {
+                    "$ref": "#/components/schemas/ResponseOutputRefusal"
+                  }
+                ]
               },
               "title": "Content",
               "type": "array"

litellm/proxy/_types.py

@@ -38,12 +38,14 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
     validate_no_callback_env_reference,
 )
 from litellm.proxy.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled
+from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
 from litellm.types.agents import (
     AgentCaller as AgentCaller,  # noqa: PLC0414  # public re-export
 )
 from litellm.types.agents import (
     AgentResponse as AgentResponse,  # noqa: PLC0414  # public re-export
 )
+from litellm.types.decisions import OpenAIDecisionResponse
 from litellm.types.integrations.compression_interception import (
     CompressionSavingsMetadata,
 )
@@ -499,6 +501,7 @@ class LiteLLMRoutes(enum.Enum):
         "/a2a/{agent_id}/message/send",
         "/v1/a2a/{agent_id}/message/send",
         "/a2a/{agent_id}/message/stream",
+        "/v1/a2a/{agent_id}/message/stream",
         "/a2a/{agent_id}/.well-known/agent-card.json",
     )
 
@@ -2470,6 +2473,22 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
     Documents all the fields supported by `general_settings` in config.yaml
     """
 
+    accounting_capture: UsageCaptureSettings | None = Field(
+        None,
+        description="Opt-in durable usage capture. Legacy spend updates remain active. Requires local Prisma writes.",
+    )
+    accounting_sync_ack: bool = Field(
+        False,
+        description="Wait for usage commit on foreground non-stream chat, text, embedding and Responses requests. "
+        "Other requests remain supported with asynchronous usage capture.",
+    )
+    accounting_sync_ack_timeout_seconds: float = Field(
+        20.0, gt=0, description="Maximum response wait for usage commit, including the batch flush interval"
+    )
+    accounting_sync_ack_failure_policy: Literal["fail_closed", "log"] = Field(
+        "fail_closed", description="Return 503 or log the persistence failure after the response wait expires"
+    )
+
     request_tag_mappings: Sequence[RequestTagMapping] | None = Field(
         None,
         description="Map W3C baggage values to request tags. Exact value/tag rules take precedence over tag_prefix fallbacks.",
@@ -4584,6 +4603,7 @@ PassThroughEndpointLoggingResultValues = (
     | StandardPassThroughResponseObject
     | ResponsesAPIResponse
     | TranscriptionResponse
+    | OpenAIDecisionResponse
 )
 
 

litellm/proxy/a2a/agent_card.py

@@ -12,6 +12,8 @@ from collections.abc import Mapping
 from copy import deepcopy
 from typing import Final, Literal
 
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
+
 SupportedA2AVersion = Literal["0.3", "1.0"]
 
 # Protocol versions LiteLLM can serve to A2A clients. The admin pins one per agent;
@@ -21,7 +23,6 @@ SUPPORTED_A2A_PROTOCOL_VERSIONS: Final[tuple[SupportedA2AVersion, ...]] = ("0.3"
 # Default served version when the agent card does not pin one.
 LITELLM_A2A_PROTOCOL_VERSION: Final = "1.0"
 
-
 _PROTOCOL_VERSION_PATTERN: Final = re.compile(
     r"^(\d+\.\d+)(?:\.\d+(?:-[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?(?:\+[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?)?$"
 )
@@ -50,6 +51,24 @@ def resolve_served_protocol_version(card: Mapping[str, object] | None) -> str:
     return normalized if normalized is not None else LITELLM_A2A_PROTOCOL_VERSION
 
 
+def point_agent_card_at_proxy(
+    agent_card: Mapping[str, object],
+    proxy_url: str,
+) -> dict[str, object]:  # mutable-ok: the public helper contract requires a dictionary
+    interfaces: Final = agent_card.get("supportedInterfaces")
+    interface_items: Final = (
+        tuple(iter_items(agent_card.get("supportedInterfaces"))) if isinstance(interfaces, list) else ()
+    )
+    first: Final[object] = interface_items[0] if interface_items else None
+    first_mapping: Final = as_str_mapping(first)
+    pointed_interfaces: Final = (
+        {"supportedInterfaces": [{**first_mapping, "url": proxy_url}, *interface_items[1:]]}
+        if isinstance(first, dict) and first_mapping is not None
+        else {}
+    )
+    return deepcopy({**agent_card, "url": proxy_url, **pointed_interfaces})
+
+
 # Security scheme exposed by the LiteLLM-fronted agent card. Always replaces
 # whatever upstream advertised — the client must authenticate to the proxy,
 # not the upstream agent.

litellm/proxy/agent_endpoints/a2a_endpoints.py

@@ -11,15 +11,15 @@ The A2A SDK can point to LiteLLM's URL and invoke agents registered with LiteLLM
 """
 
 import json
-from collections.abc import AsyncGenerator, Mapping
-from copy import deepcopy
+from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Protocol
+from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
 from urllib.parse import urlparse
 
 from fastapi import APIRouter, Depends, HTTPException, Request, Response
 from fastapi.responses import JSONResponse, StreamingResponse
-from pydantic import TypeAdapter, ValidationError
+from fastapi.routing import APIRoute
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -27,7 +27,8 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
 from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header
 from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy._types import ProxyException, UserAPIKeyAuth
+from litellm.proxy.a2a.agent_card import point_agent_card_at_proxy
 from litellm.proxy.a2a.version_convert import (
     A2AVersion,
     normalize_agent_card,
@@ -42,6 +43,8 @@ from litellm.proxy.agent_endpoints.databricks_oauth import (
 )
 from litellm.proxy.agent_endpoints.utils import merge_agent_headers
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body
+from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception
 from litellm.proxy.common_utils.sse_keepalive import (
     SSE_COMMENT_PING,
     coerce_keepalive_interval,
@@ -57,6 +60,42 @@ if TYPE_CHECKING:
 
 router: Final = APIRouter()
 
+
+class _A2AJsonRpcRequest(BaseModel):
+    model_config = ConfigDict(extra="allow", frozen=True)
+
+    jsonrpc: Literal["2.0"]
+    id: str | int | None = None
+    method: str
+    params: Mapping[str, object]
+
+
+class _A2AJsonRpcResponse(BaseModel):
+    model_config = ConfigDict(extra="allow", frozen=True)
+
+    jsonrpc: Literal["2.0"]
+    id: str | int | None
+    result: object | None = None
+    error: Mapping[str, object] | None = None
+
+
+_A2A_MESSAGE_STREAM_REQUEST_BODY: Final = inline_request_body(
+    _A2AJsonRpcRequest,
+    {
+        "jsonrpc": "2.0",
+        "id": "stream-1",
+        "method": "message/stream",
+        "params": {
+            "message": {
+                "messageId": "message-1",
+                "role": "user",
+                "parts": [{"kind": "text", "text": "Hello"}],
+                "kind": "message",
+            }
+        },
+    },
+)
+
 # Mirrors the native seam's own headers: a reverse proxy that batches the whole
 # stream would swallow the keepalives this route sends to defeat idle timeouts.
 _SSE_KEEPALIVE_HEADERS: Final[Mapping[str, str]] = MappingProxyType(
@@ -172,13 +211,16 @@ def _caller_identity_headers(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str,
 async def _resolve_backend_auth(
     litellm_params: dict[str, object],
     custom_llm_provider: object,
+    agent_extra_headers: Mapping[str, str] | None = None,
 ) -> AgentBackendAuth:
     databricks_auth: Final = await resolve_databricks_backend_auth(litellm_params)
     if has_databricks_oauth(litellm_params):
         return databricks_auth
     return AgentBackendAuth(
         litellm_params=databricks_auth.litellm_params,
-        header=await resolve_a2a_hop_auth_header(litellm_params, custom_llm_provider),
+        header=await resolve_a2a_hop_auth_header(
+            litellm_params, custom_llm_provider, agent_extra_headers=agent_extra_headers
+        ),
     )
 
 
@@ -201,6 +243,35 @@ def _forwarding_headers(
     return merged or None
 
 
+async def _log_agent_rejection(
+    *,
+    proxy_logging_obj: ProxyLogging,
+    user_api_key_dict: UserAPIKeyAuth,
+    request_id: object,
+    method: str | None,
+    rejection: HTTPException,
+    target_agent_id: str,
+    model: str,
+) -> None:
+    request_data: Final[dict[str, object]] = {  # mutable-ok: the failure hook stamps its logging state onto it
+        "jsonrpc": "2.0",
+        "id": request_id,
+        "method": method,
+        "model": model,
+        "agent_id": target_agent_id,
+        "metadata": {"agent_id": target_agent_id},
+    }
+    try:
+        await proxy_logging_obj.post_call_failure_hook(  # pyright: ignore[reportUnknownMemberType]  # bare dict in sig
+            request_data=request_data,
+            original_exception=rejection,
+            user_api_key_dict=user_api_key_dict,
+            route="/a2a/{agent_id}",
+        )
+    except Exception:  # noqa: BLE001  # a logging failure must not change the rejection the caller gets
+        verbose_proxy_logger.exception("Failed to log rejected A2A request")
+
+
 def _jsonrpc_error(
     request_id: object,
     code: int,
@@ -218,6 +289,57 @@ def _jsonrpc_error(
     )
 
 
+class _JsonRpcRequestId(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    id: str | int | None = None
+
+
+async def _jsonrpc_request_id(request: Request) -> str | int | None:
+    try:
+        return _JsonRpcRequestId.model_validate_json(await request.body()).id
+    except ValidationError:
+        return None
+
+
+async def _jsonrpc_refusal(request: Request, refusal: ProxyException | HTTPException) -> JSONResponse:
+    from litellm.proxy.proxy_server import (
+        _close_dangling_otel_server_span,  # pyright: ignore[reportPrivateUsage]  # proxy_server keeps the span-close helper private; a route that answers an error itself must stamp the OTel server span like the global ProxyException handler does
+    )
+
+    error: Final = refusal if isinstance(refusal, ProxyException) else status_typed_proxy_exception(refusal)
+    _close_dangling_otel_server_span(request, int(error.code), exc=refusal)
+    return JSONResponse(
+        content={
+            "jsonrpc": "2.0",
+            "id": await _jsonrpc_request_id(request),
+            "error": {
+                "code": -32000,
+                "message": error.message,
+                "data": {"type": error.type, "param": error.param, "code": error.code},
+            },
+        },
+        status_code=int(error.code),
+        headers=error.headers if isinstance(refusal, ProxyException) else refusal.headers,
+    )
+
+
+class _JsonRpcRefusalRoute(APIRoute):
+    def get_route_handler(self) -> Callable[[Request], Coroutine[object, object, Response]]:
+        handler: Final = super().get_route_handler()
+
+        async def answer_refusals_as_jsonrpc(request: Request) -> Response:
+            try:
+                return await handler(request)
+            except (ProxyException, HTTPException) as refusal:
+                return await _jsonrpc_refusal(request, refusal)
+
+        return answer_refusals_as_jsonrpc
+
+
+jsonrpc_router: Final = APIRouter(route_class=_JsonRpcRefusalRoute)
+
+
 async def _get_agent(agent_id: str) -> "AgentResponse | None":
     """Look up an agent by ID or name. Returns None if not found."""
     from litellm.proxy.common_utils.registry_read_through import (
@@ -628,16 +750,12 @@ async def get_agent_card(
             )
 
         proxy_url: Final = get_custom_url(str(request.base_url), route=f"a2a/{agent_id}")
-        agent_card = deepcopy(agent.agent_card_params)
-        agent_card["url"] = proxy_url
-        interfaces: Final = agent_card.get("supportedInterfaces")
-        if isinstance(interfaces, list) and interfaces:
-            interfaces[0]["url"] = proxy_url
+        agent_card: Final = point_agent_card_at_proxy(agent.agent_card_params, proxy_url)
         served_version: Final = _served_version(agent, request)
-        agent_card = normalize_agent_card(agent_card, served_version)
+        normalized_agent_card: Final = normalize_agent_card(agent_card, served_version)
 
         verbose_proxy_logger.debug("Returning agent card for '%s' with proxy URL: %s", agent_id, proxy_url)
-        return JSONResponse(content=agent_card)
+        return JSONResponse(content=normalized_agent_card)
 
     except HTTPException:
         raise
@@ -646,21 +764,35 @@ async def get_agent_card(
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@router.post(
+@jsonrpc_router.post(
     "/a2a/{agent_id}",
     tags=["[beta] A2A Agents"],
     dependencies=[Depends(user_api_key_auth)],
 )
-@router.post(
+@jsonrpc_router.post(
     "/a2a/{agent_id}/message/send",
     tags=["[beta] A2A Agents"],
     dependencies=[Depends(user_api_key_auth)],
 )
-@router.post(
+@jsonrpc_router.post(
     "/v1/a2a/{agent_id}/message/send",
     tags=["[beta] A2A Agents"],
     dependencies=[Depends(user_api_key_auth)],
 )
+@router.post(
+    "/a2a/{agent_id}/message/stream",
+    tags=["[beta] A2A Agents"],
+    dependencies=[Depends(user_api_key_auth)],
+    openapi_extra={"requestBody": _A2A_MESSAGE_STREAM_REQUEST_BODY},
+    responses={200: {"model": _A2AJsonRpcResponse}},
+)
+@router.post(
+    "/v1/a2a/{agent_id}/message/stream",
+    tags=["[beta] A2A Agents"],
+    dependencies=[Depends(user_api_key_auth)],
+    openapi_extra={"requestBody": _A2A_MESSAGE_STREAM_REQUEST_BODY},
+    responses={200: {"model": _A2AJsonRpcResponse}},
+)
 async def invoke_agent_a2a(
     agent_id: str,
     request: Request,
@@ -722,6 +854,15 @@ async def invoke_agent_a2a(
         # Find the agent
         agent: Final = await _get_agent(agent_id)
         if agent is None:
+            await _log_agent_rejection(
+                proxy_logging_obj=proxy_logging_obj,
+                user_api_key_dict=user_api_key_dict,
+                request_id=request_id,
+                method=method,
+                rejection=HTTPException(status_code=404, detail=f"Agent '{agent_id}' not found"),
+                target_agent_id=agent_id,
+                model=f"a2a_agent/{agent_id}",
+            )
             return _jsonrpc_error(request_id, -32000, f"Agent '{agent_id}' not found", 404)
 
         served_version: Final = _served_version(agent, request, original_method)
@@ -731,10 +872,20 @@ async def invoke_agent_a2a(
             user_api_key_auth=user_api_key_dict,
         )
         if not is_allowed:
-            raise HTTPException(
+            denial: Final = HTTPException(
                 status_code=403,
                 detail=f"Agent '{agent_id}' is not allowed for your key/team. Contact proxy admin for access.",
             )
+            await _log_agent_rejection(
+                proxy_logging_obj=proxy_logging_obj,
+                user_api_key_dict=user_api_key_dict,
+                request_id=request_id,
+                method=method,
+                rejection=denial,
+                target_agent_id=agent.agent_id,
+                model=f"a2a_agent/{(as_str_mapping(agent.agent_card_params or {}) or {}).get('name', agent_id)}",
+            )
+            raise denial
 
         user_api_key_dict.invoked_agent_id = agent.agent_id
         strict_policy: Final = agent.identity_managed or bool(user_api_key_dict.requires_fresh_policy)
@@ -834,14 +985,15 @@ async def invoke_agent_a2a(
                     if header_name:
                         dynamic_headers[header_name] = val
 
-        backend_auth: Final = await _resolve_backend_auth(litellm_params, custom_llm_provider)
+        merged_agent_headers: Final = merge_agent_headers(
+            dynamic_headers=dynamic_headers or None,
+            static_headers=static_headers or None,
+        )
+        backend_auth: Final = await _resolve_backend_auth(litellm_params, custom_llm_provider, merged_agent_headers)
         agent_extra_headers: Final = _forwarding_headers(
             caller_identity=caller_identity,
             request_data=data,
-            agent_extra_headers=merge_agent_headers(
-                dynamic_headers=dynamic_headers or None,
-                static_headers=static_headers or None,
-            ),
+            agent_extra_headers=merged_agent_headers,
             backend_auth_header=backend_auth.header,
         )
         forwarded_litellm_params: Final = backend_auth.litellm_params
@@ -1051,3 +1203,6 @@ async def invoke_agent_a2a(
         if isinstance(e, litellm.BadRequestError):
             return _jsonrpc_error(body.get("id"), -32602, e.message, 400)
         return _jsonrpc_error(body.get("id"), -32603, f"Internal error: {e}", 500)
+
+
+router.include_router(jsonrpc_router)

litellm/proxy/agent_endpoints/agent_search.py

@@ -140,7 +140,7 @@ async def search_agents(
     user_api_key_dict: UserAPIKeyAuth,
     proxy_logging_obj: ProxyLogging,
 ) -> AgentSearchOutcome:
-    if embedding_model is None:
+    if embedding_model is None or not embedding_model.strip():
         return AgentSearchNotConfigured(
             reason="agent search needs litellm_settings.agent_search_embedding_model set to an embedding model from model_list"
         )

litellm/proxy/agent_endpoints/endpoints.py

@@ -22,6 +22,7 @@ from typing_extensions import ReadOnly, Required, assert_never
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import AGENT_HEALTH_CHECK_GATHER_TIMEOUT_SECONDS, AGENT_HEALTH_CHECK_TIMEOUT_SECONDS
+from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.proxy._types import (
     CommonProxyErrors,
@@ -33,6 +34,7 @@ from litellm.proxy.a2a.agent_card import (
     SUPPORTED_A2A_PROTOCOL_VERSIONS,
     merge_agent_card,
     normalize_protocol_version,
+    point_agent_card_at_proxy,
 )
 from litellm.proxy.agent_endpoints.agent_registry import (
     AgentIdWhere,
@@ -180,10 +182,36 @@ def _redact_agent_litellm_params_dict(
     return dict(parse_agent_litellm_params(redact_sensitive_agent_litellm_params(litellm_params)))
 
 
+def _non_admin_agent_card(
+    card: Mapping[str, object], proxy_url: str
+) -> dict[str, object]:  # mutable-ok: AgentResponse.agent_card_params requires a mutable dictionary
+    pointed_card: Final = point_agent_card_at_proxy(card, proxy_url)
+    interfaces: Final = pointed_card.get("supportedInterfaces")
+    interface_items: Final = (
+        tuple(iter_items(pointed_card.get("supportedInterfaces"))) if isinstance(interfaces, list) else ()
+    )
+    first_interface: Final[object] = interface_items[0] if interface_items else None
+    first_mapping: Final = as_str_mapping(first_interface)
+    proxy_interface: Final = (
+        {"supportedInterfaces": [{**first_mapping, "url": proxy_url}]}
+        if isinstance(first_interface, dict) and first_mapping is not None and first_mapping.get("url") == proxy_url
+        else {}
+    )
+    return {
+        **{
+            key: value
+            for key, value in pointed_card.items()
+            if key not in ("additionalInterfaces", "supportedInterfaces")
+        },
+        **proxy_interface,
+    }
+
+
 def _redact_sensitive_agent_fields(
     agents: Sequence[AgentResponse],
     *,
     is_admin: bool,
+    request_base_url: str,
 ) -> list[AgentResponse]:
     """
     Return copies of the given agents with credential-bearing litellm_params
@@ -207,6 +235,11 @@ def _redact_sensitive_agent_fields(
             copy.keys = None
             copy.kill_switch = None
             copy.identity = None
+            if copy.agent_card_params:
+                copy.agent_card_params = _non_admin_agent_card(
+                    copy.agent_card_params,
+                    get_custom_url(request_base_url, route=f"a2a/{copy.agent_id}"),
+                )
         if copy.litellm_params:
             copy.litellm_params = _redact_agent_litellm_params_dict(copy.litellm_params)
         copy.kill_switch = redact_kill_switch(copy.kill_switch)
@@ -461,14 +494,6 @@ async def get_agents(
                 global_agent_registry.ids_for_agent(agent.agent_id).isdisjoint(litellm.public_agent_groups)
             )
 
-        # litellm_params secrets are always redacted; keys/headers stay
-        # admin-only.
-        is_admin: Final = (
-            user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
-            or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
-        )
-        returned_agents = _redact_sensitive_agent_fields(returned_agents, is_admin=is_admin)
-
         if health_check:
             agents_with_url: Final = [agent for agent in returned_agents if (agent.agent_card_params or {}).get("url")]
             agents_without_url = [agent for agent in returned_agents if not (agent.agent_card_params or {}).get("url")]
@@ -493,6 +518,18 @@ async def get_agents(
             healthy_ids: Final = {result["agent_id"] for result in health_results if result["healthy"]}
             returned_agents = [agent for agent in agents_with_url if agent.agent_id in healthy_ids] + agents_without_url
 
+        # litellm_params secrets are always redacted; keys/headers stay
+        # admin-only.
+        is_admin: Final = (
+            user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
+            or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
+        )
+        returned_agents = _redact_sensitive_agent_fields(
+            returned_agents,
+            is_admin=is_admin,
+            request_base_url=str(request.base_url),
+        )
+
         if query is None:
             return returned_agents
         return await _rank_agents_by_query(query, returned_agents, top_k, user_api_key_dict)
@@ -691,7 +728,11 @@ async def create_agent(
 
         # The caller is a proxy admin (enforced above); litellm_params
         # secrets are still never echoed back in the response.
-        return _redact_sensitive_agent_fields((result,), is_admin=True)[0]
+        return _redact_sensitive_agent_fields(
+            (result,),
+            is_admin=True,
+            request_base_url=str(http_request.base_url),
+        )[0]
 
     except HTTPException:
         raise
@@ -708,6 +749,7 @@ async def create_agent(
 )
 async def get_agent_by_id(
     agent_id: str,
+    request: Request,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
 ):
     """
@@ -770,7 +812,11 @@ async def get_agent_by_id(
             user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN
             or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
         )
-        agent = _redact_sensitive_agent_fields((agent,), is_admin=is_admin)[0]
+        agent = _redact_sensitive_agent_fields(
+            (agent,),
+            is_admin=is_admin,
+            request_base_url=str(request.base_url),
+        )[0]
 
         return agent
     except HTTPException:
@@ -879,7 +925,11 @@ async def update_agent(
             "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id
         )
 
-        return _redact_sensitive_agent_fields((result,), is_admin=True)[0]
+        return _redact_sensitive_agent_fields(
+            (result,),
+            is_admin=True,
+            request_base_url=str(http_request.base_url),
+        )[0]
     except HTTPException:
         raise
     except Exception as e:
@@ -987,7 +1037,11 @@ async def patch_agent(
             "Successfully updated agent '%s' (ID: %s) in memory", existing_agent.get("agent_name"), agent_id
         )
 
-        return _redact_sensitive_agent_fields((result,), is_admin=True)[0]
+        return _redact_sensitive_agent_fields(
+            (result,),
+            is_admin=True,
+            request_base_url=str(http_request.base_url),
+        )[0]
     except HTTPException:
         raise
     except Exception as e:

litellm/proxy/auth/auth_checks.py

@@ -1086,13 +1086,31 @@ def model_max_budget_of(holder: ModelMaxBudgetHolder) -> Mapping[str, object] |
 
 
 def route_skips_budget_checks(route: str) -> bool:
-    return route not in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES and (
-        route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route)
+    if route in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES:
+        return False
+    if route in MODEL_DISCOVERY_ROUTES:
+        return True
+    if RouteChecks.is_llm_api_route(route=route):
+        return False
+    from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (  # noqa: PLC0415  # pass-through endpoints import the auth stack
+        InitPassThroughEndpointHelpers,
     )
 
+    return not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=route)
 
-def request_skips_budget_checks(route: str, model: str | list[str] | None, llm_router: Router | None) -> bool:
-    return route_skips_budget_checks(route=route) or is_model_cost_zero(model=model, llm_router=llm_router)
+
+def request_skips_budget_checks(
+    route: str,
+    model: str | list[str] | None,
+    llm_router: Router | None,
+    *,
+    request_body: Mapping[str, object] | None = None,
+) -> bool:
+    if route_skips_budget_checks(route=route):
+        return True
+    if request_body is not None and is_mcp_discovery_request(route=route, request_body=request_body):
+        return True
+    return is_model_cost_zero(model=model, llm_router=llm_router)
 
 
 async def common_checks(
@@ -1142,10 +1160,11 @@ async def common_checks(
         team_id=valid_token.team_id if valid_token is not None else None,
     )
 
-    skip_all_budget_checks: Final = (
-        skip_budget_checks
-        or route_skips_budget_checks(route=route)
-        or is_mcp_discovery_request(route=route, request_body=_typed_request_body(request_body))
+    skip_all_budget_checks: Final = skip_budget_checks or request_skips_budget_checks(
+        route=route,
+        model=_model,
+        llm_router=llm_router,
+        request_body=_typed_request_body(request_body),
     )
 
     membership_user_id: Final = (
@@ -5359,79 +5378,6 @@ async def _granted_model_lists(
     )
 
 
-async def _user_object_or_none(
-    valid_token: UserAPIKeyAuth,
-    prisma_client: PrismaClient,
-    user_api_key_cache: UserApiKeyCache,
-    proxy_logging_obj: ProxyLogging,
-) -> LiteLLM_UserTable | None:
-    try:
-        return await get_user_object(
-            user_id=valid_token.user_id,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            user_id_upsert=False,
-            proxy_logging_obj=proxy_logging_obj,
-        )
-    except UserNotFoundError:
-        return None
-
-
-async def enforced_model_allowlists(
-    valid_token: UserAPIKeyAuth,
-    prisma_client: PrismaClient | None,
-    user_api_key_cache: UserApiKeyCache,
-    proxy_logging_obj: ProxyLogging,
-) -> tuple[Sequence[str], ...]:
-    """One model allowlist per level that ``common_checks`` enforces on a request from this identity."""
-    key_models: Final = _resolve_key_models_for_auth_check(valid_token=valid_token)
-    if prisma_client is None:
-        return (key_models, tuple(valid_token.team_models or ()))
-    team_object: Final = (
-        None
-        if valid_token.team_id is None
-        else await get_team_object(
-            team_id=valid_token.team_id,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            proxy_logging_obj=proxy_logging_obj,
-        )
-    )
-    user_object: Final = (
-        None
-        if team_object is not None
-        else await _user_object_or_none(
-            valid_token=valid_token,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            proxy_logging_obj=proxy_logging_obj,
-        )
-    )
-    project_object: Final = (
-        None
-        if valid_token.project_id is None
-        else await get_project_object(
-            project_id=valid_token.project_id,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            proxy_logging_obj=proxy_logging_obj,
-        )
-    )
-    return (
-        key_models,
-        team_object.models if team_object is not None else (),
-        await _team_member_granted_models(
-            valid_token=valid_token,
-            team_object=team_object,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            proxy_logging_obj=proxy_logging_obj,
-        ),
-        user_object.models if user_object is not None else (),
-        project_object.models if project_object is not None else (),
-    )
-
-
 async def collect_matched_model_access_groups(
     model: str | Sequence[str] | None,
     valid_token: UserAPIKeyAuth | None,

litellm/proxy/auth/user_api_key_auth.py

@@ -80,7 +80,6 @@ from litellm.proxy.auth.auth_checks import (  # noqa: F401  # legacy module expo
     get_team_object,
     get_user_object,
     get_user_role,
-    is_mcp_discovery_request,
     is_model_cost_zero,
     is_user_proxy_admin,
     is_valid_fallback_model,
@@ -88,6 +87,7 @@ from litellm.proxy.auth.auth_checks import (  # noqa: F401  # legacy module expo
     key_access_group_grants_model,
     key_model_aliases_for_auth_check,
     model_max_budget_of,
+    request_skips_budget_checks,
     resolve_and_validate_end_user_id,
     resolve_default_end_user_budget,
     route_skips_budget_checks,
@@ -2497,15 +2497,9 @@ async def user_api_key_auth_builder(
                             llm_router=llm_router,
                             team_id=valid_token.team_id,
                         )
-                        skip_budget_checks = is_mcp_discovery_request(route=route, request_body=request_data)
-                        if not skip_budget_checks and model is not None and llm_router is not None:
-                            from litellm.proxy.auth.auth_checks import is_model_cost_zero
-
-                            skip_budget_checks = (  # rebind-ok: pre-existing rebinding on a rename-only line
-                                is_model_cost_zero(model=model, llm_router=llm_router)
-                            )
-                            if skip_budget_checks:
-                                verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model)
+                        skip_budget_checks = request_skips_budget_checks(
+                            route=route, model=model, llm_router=llm_router, request_body=request_data
+                        )
 
                         # Fetch project object for JWT path if project_id is set
                         _jwt_project_obj = None
@@ -3019,15 +3013,9 @@ async def validate_resolved_virtual_key(  # noqa: C901  # Preserve ordering of e
             llm_router=llm_router,
             team_id=valid_token.team_id,
         )
-        skip_budget_checks = is_mcp_discovery_request(route=route, request_body=request_data)
-        if not skip_budget_checks and model is not None and llm_router is not None:
-            from litellm.proxy.auth.auth_checks import is_model_cost_zero
-
-            skip_budget_checks = is_model_cost_zero(  # rebind-ok: pre-existing rebinding on a rename-only line
-                model=model, llm_router=llm_router
-            )
-            if skip_budget_checks:
-                verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model)
+        skip_budget_checks: Final = request_skips_budget_checks(
+            route=route, model=model, llm_router=llm_router, request_body=request_data
+        )
 
         # Check 3. Check if user is in their team budget
         if not skip_budget_checks and valid_token.team_member_spend is not None:
@@ -4088,11 +4076,7 @@ def _should_skip_budget_checks(
         llm_router=llm_router,
         team_id=team_id,
     )
-    if is_mcp_discovery_request(route=route, request_body=request_data):
-        return True
-    if model is not None and llm_router is not None:
-        return is_model_cost_zero(model=model, llm_router=llm_router)
-    return False
+    return request_skips_budget_checks(route=route, model=model, llm_router=llm_router, request_body=request_data)
 
 
 def _resolve_request_principal(request: Request, valid_token: UserAPIKeyAuth) -> Principal:

litellm/proxy/common_request_processing.py

@@ -137,6 +137,7 @@ from litellm.proxy.route_llm_request import (
     route_request,
 )
 from litellm.proxy.spend_tracking.cost_finalization import finalize_proxy_response_cost, prepare_response_cost
+from litellm.proxy.spend_tracking.usage_capture import capture_proxy_response
 from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id
 from litellm.proxy.utils import (  # noqa: F401  # legacy module exports
     ProxyLogging,
@@ -3366,6 +3367,9 @@ class ProxyBaseLLMRequestProcessing:
                     )
                 ),
             )
+            await capture_proxy_response(
+                logging_obj, response, background=_REQUEST_FIELDS.validate_python(self.data).get("background") is True
+            )
         except (Exception, asyncio.CancelledError) as exc:
             _exception_raised = True
             if logging_obj.prepared_cost is not None:

litellm/proxy/common_utils/realtime_utils.py

@@ -1,3 +1,4 @@
+import json
 from functools import lru_cache
 from typing import Final
 
@@ -10,7 +11,7 @@ def realtime_request_body(model: str | None) -> bytes:
     Generate the realtime websocket request body. Cached with LRU semantics to avoid repeated
     string formatting work while keeping memory usage bounded.
     """
-    return f'{{"model": "{model or ""}"}}'.encode()
+    return json.dumps({"model": model or ""}).encode()
 
 
 _realtime_request_body: Final = realtime_request_body