Merge remote-tracking branch 'origin/main' into HEAD

# Conflicts:
#	litellm/router.py

.github/PULL_REQUEST_TEMPLATE/general.md

@@ -4,9 +4,14 @@
      Drop every section you have nothing to put in, heading included: a bare "## Relevant issues" or
      "## Affected release" with nothing under it must not appear in the final description -->
 
-<!-- Two or three plain sentences a teammate can read in 10 seconds, written the way you'd explain it in Slack
+## Handwritten 1-3 sentence description
+
+<!-- The PR author rewrites this in their own words before asking for review. Being able to sum up the PR in a few
+     plain sentences proves the author understands what it does, and human-written technical English is far easier
+     for a reviewer to parse than AI-written English
+     An AI agent may draft a starting point here, but the author replaces it with their own handwritten version
      Say what the PR does for users and why, then anything that keeps it working over time (a sync job, a config someone owns)
-     No headings, bullets, tables, or code names here. Everything else goes inside the Details block below -->
+     No bullets, tables, or code names here. Everything else goes inside the Details block below -->
 
 <details>
 <summary>Details</summary>

.github/workflows/create-rc-branch.yml

@@ -2,7 +2,7 @@ name: Create RC Branch
 
 on:
   schedule:
-    - cron: "0 3 * * 5"
+    - cron: "0 3 * * 1"
       timezone: "America/Los_Angeles"
   workflow_dispatch:
 

enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py

@@ -200,6 +200,7 @@ async def log_billed_background_response(billed: BilledBackgroundResponse) -> Bi
     unpriced: Final = _without_read_price(billed.response)
     cost: Final = logging_obj.response_cost_calculator(result=unpriced)
     end_time: Final = datetime.now()
+    logging_obj.capture_usage_before_logging(unpriced, logging_obj.start_time, end_time)
     GLOBAL_LOGGING_WORKER.ensure_initialized_and_enqueue(
         async_coroutine=logging_obj.async_success_handler(
             result=unpriced, start_time=logging_obj.start_time, end_time=end_time

litellm-rust/crates/cost/AGENTS.md

@@ -0,0 +1 @@
+The price vocabulary (meter, band, channel, `When`, `Billing`) is explained in [../model-catalog/AGENTS.md](../model-catalog/AGENTS.md#concepts), the only crate that maps it to JSON keys

litellm-rust/crates/cost/src/price.rs

@@ -19,6 +19,7 @@ pub enum ServiceTier {
     Balanced,
 }
 
+/// A prompt-size range. `OverPromptTokens(n)` is spelled `_above_<n>_tokens` or `_above_<n / 1000>k_tokens`
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum Band {
     #[default]
@@ -26,12 +27,15 @@ pub enum Band {
     OverPromptTokens(NonZeroU64),
 }
 
+/// The request condition a price applies under, its service tier and prompt band
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct When {
     pub tier: ServiceTier,
     pub band: Band,
 }
 
+/// The route whose price list a call reads, from the `_batches` and `_decisions` suffixes. Not part of `When`,
+/// since a missing batch price is half the online rate and a missing decisions price is the full online rate
 #[derive(Clone, Copy, Debug, Default, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum Channel {
     #[default]
@@ -40,6 +44,7 @@ pub enum Channel {
     Decisions,
 }
 
+/// The key of one rate, what is counted, through which channel and under which condition
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub struct Price {
     pub meter: Meter,
@@ -47,6 +52,7 @@ pub struct Price {
     pub when: When,
 }
 
+/// Every rate a row lists, its off-peak overlay and the row attributes that say how the rates apply
 #[derive(Clone, Debug, Default, PartialEq)]
 pub struct Prices {
     rates: BTreeMap<Price, UnitPrice>,
@@ -117,6 +123,7 @@ pub enum TierScope {
     WholeCall,
 }
 
+/// Rates that replace the row's token rates while a UTC window in the schedule is open
 #[derive(Clone, Debug, PartialEq)]
 pub struct OffPeak {
     schedule: Schedule,
@@ -133,6 +140,7 @@ pub struct OffPeakRates {
     pub reasoning: Option<UnitPrice>,
 }
 
+/// Multipliers applied on top of a priced call, never prices themselves
 #[derive(Clone, Debug, Default, PartialEq)]
 pub struct Adjustments(BTreeMap<Uplift, Multiplier>);
 

litellm-rust/crates/cost/src/price/meter.rs

@@ -1,5 +1,6 @@
 use std::num::NonZeroU64;
 
+/// What a price counts. A variation only one meter has, such as cache TTL or image size, lives inside its variant
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum Meter {
     Input(InputUnit),
@@ -51,12 +52,14 @@ pub enum OutputUnit {
     VideoSecond,
 }
 
+/// `Explicit` is a cache the caller created ahead of time, spelled `explicit_cache_read_*`. `Implicit` is every other read
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum CacheReadMode {
     Implicit,
     Explicit,
 }
 
+/// How long a cache write lives. `OneHour` and `TwelveHours` are spelled `_above_1hr` and `_above_12hr`
 #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
 pub enum CacheTtl {
     Default,

litellm-rust/crates/model-catalog/AGENTS.md

@@ -1,3 +1,41 @@
+## Concepts
+
+Every price in a row lowers to one `Price { meter, channel, when }` and its `UnitPrice`. These keys come from the `FlatKey::parse` tests
+
+- `input_cost_per_token`
+  - meter `Input(Token(Text))`, channel `Online`, tier `Standard`, band `Base`
+- `output_cost_per_token_above_272k_tokens_priority`
+  - meter `Output(Token(Text))`, channel `Online`
+  - `_priority` sets tier `Priority`
+  - `_above_272k_tokens` sets band `OverPromptTokens(272000)`
+- `cache_creation_input_token_cost_above_1hr_above_200k_tokens`
+  - meter `CacheWrite { Text, OneHour }`, where `_above_1hr` is the cache TTL, not a band
+  - `_above_200k_tokens` sets band `OverPromptTokens(200000)`
+  - channel `Online`, tier `Standard`
+- `ocr_cost_per_page_batches`
+  - meter `Feature(OcrPage)`, tier `Standard`, band `Base`
+  - `_batches` sets channel `Batch`
+
+- Meter is what gets counted, such as text input tokens, reasoning tokens, an image of a given size or a web search
+  - A variation only one meter has stays inside it (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`)
+- `UnitPrice` is USD for one unit of the meter, such as one token, image, page or second
+- Band is a range of prompt sizes, starting where `_above_<N>_tokens` or `_above_<N>k_tokens` says
+  - `Prices::band` picks one band from prompt tokens alone, the highest edge passed wins, and `Band::Base` is below every edge
+  - The whole call moves, so every token meter, output included, bills all its tokens at that band's rate
+  - `Billing::band_edge` says whether a prompt exactly at the edge is in the band (exclusive by default, inclusive for xAI)
+  - A band only another tier or the batch channel prices never moves the call
+- `When` is the request's condition, its service tier and band
+  - A missing price falls back from `(tier, band)` to `(standard, band)`, `(tier, base)`, then `(standard, base)`
+- `Channel` is the route whose price list the call reads, and each falls back differently
+  - `Online` falls back across `When` as above
+  - `Batch` (`_batches`) uses bands but no service tier, and a missing price is half the online rate
+  - `Decisions` (`_decisions`, OpenAI `/v1/decisions`) overrides single online fields, and a missing price is the full online rate
+- Off-peak and `Adjustments` are never prices
+  - Off-peak swaps in other token rates while a UTC window is open
+  - `Adjustments` multiplies a priced call for region, residency, geo or `speed: fast`
+- `Billing` on `Prices` holds row attributes that change how its prices apply (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope)
+  - `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list
+
 ## Layout
 
 `wire/` reads one entry as written. `EntryWire::read` makes one pass over the entry's keys and puts every non-null key in exactly one place, which is an attribute, a flat price parsed by `FlatKey::parse`, a price block, a `supports_*` capability or `unknown`. A known key with a bad value becomes an `Error` that names the key, and the rest of the entry is still read
@@ -6,8 +44,6 @@
 
 `catalog.rs` reads the document, indexes aliases and runs the checks across entries. A bad entry keeps its raw fields and its errors are listed in `Loaded.errors`. Deciding what to do with them is the caller's job. `Catalog::written` gives back every top-level key and value as the document wrote them, in document order. `Catalog::served` gives back every model and alias as Python serves it, without `sample_spec` and `fallback_generalizations`
 
-`Billing` on `Prices` holds the row attributes pricing reads (text unit, band edge, cache-read fallback, wall clock, web search unit, tier scope). `lower.rs` derives it from the row and its `litellm_provider`, so `litellm-cost` holds no provider list
-
 `store/` is the catalog's in-memory store and the only place that indexes model names. Its rules are in [src/store/AGENTS.md](src/store/AGENTS.md)
 
 `layers.rs`, `resolve/`, `model_info.rs` and `register.rs` hold types only. The functions over them land with their first caller
@@ -16,7 +52,7 @@ Catalog errors use one opaque `Error` with typed causes and JSON Pointer accesso
 
 ## Prices
 
-The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names. Every price is keyed by `Meter`, `Channel` and `When`. A variation only one meter has stays in the meter (cache TTL, image size, resolution, `_above_Ns_interval`, `max_results_range`). Service tier and prompt band make up `When`. Batch and decisions are a `Channel` because Python bills them with their own rules. Off-peak is an overlay on `Prices`, and regional uplifts are `Adjustments`, never prices
+The price vocabulary lives in `litellm-cost`, and only this crate knows JSON key names
 
 A new `N`, image size or resolution in an existing key shape parses with no Rust change. A new unit, token kind, feature or tier needs a variant in `litellm-cost` plus its spelling in `wire/flat_key.rs` and `FLAT_KEY_PATTERN` in `schema.rs`
 

litellm/_internal_context.py

@@ -21,6 +21,7 @@ _T = TypeVar("_T")
 # When True, suppresses async logging and billing for internal sub-calls
 # (e.g., emulated file-search steps that make nested LLM calls).
 is_internal_call: Final[ContextVar[bool]] = ContextVar("is_internal_call", default=False)
+internal_parent_request_id: Final[ContextVar[str | None]] = ContextVar("internal_parent_request_id", default=None)
 
 # One request prices its totals, its per-token-type lines and the rates it reports on
 # separate code paths. Each reads the clock for off-peak pricing, so without a pinned
@@ -40,13 +41,15 @@ _mirrored_call: Final[ContextVar[bool]] = ContextVar("mirrored_call", default=Fa
 
 
 @contextmanager
-def internal_sub_call() -> Generator[None]:
+def internal_sub_call(parent_request_id: str | None = None) -> Generator[None]:
     """A nested call the parent call logs and bills, so the parent must fold this call's cost into its own."""
     token: Final = is_internal_call.set(True)
+    parent_token: Final = internal_parent_request_id.set(parent_request_id or internal_parent_request_id.get())
     try:
         yield
     finally:
         is_internal_call.reset(token)
+        internal_parent_request_id.reset(parent_token)
 
 
 @contextmanager

litellm/accounting/protocols.py

@@ -3,7 +3,7 @@
 The event row's ``payload`` is a private envelope versioned by the row's ``schema_version``.
 Usage envelopes retain the public event and request-scoped pricing context: quantities with units,
 model/provider, requested and billed tier/region when known, and per-request/deployment overrides.
-These inputs must survive request teardown. Cost-map rates are resolved at compilation, not capture.
+These inputs, including the effective cost-map rates, must survive request teardown.
 Cost envelopes retain the public CostEvent, including its required finalized pricing snapshot.
 T5/T6 own envelope serialization; this public API does not introduce a second persisted schema.
 """
@@ -38,8 +38,8 @@ class CostCompiler(Protocol):
     async def compile(self, event: OutboundUsageEvent) -> CostEvent | Unpriced:
         """Reuse the committed charge or finalize one charge and its delivery intent.
 
-        Resolve captured request-scoped context by event identity and cost-map rates
-        at compilation. Persist the resolved inputs in the required snapshot. Replay
+        Resolve captured request-scoped context and rates by event identity.
+        Persist the resolved inputs in the required snapshot. Replay
         reuses the committed result and never consumes free allowance again.
         Missing pricing is Unpriced, never an inferred zero.
         """

litellm/accounting/types.py

@@ -276,8 +276,8 @@ class NamedCost(BaseModel):
 class PricingSnapshot(BaseModel):
     """Retained inputs and result of authoritative compilation, reproducible after cost-map rates change.
 
-    Request-scoped inputs and overrides come from the usage row's versioned payload. Cost-map rates are
-    resolved at compilation, not frozen at capture. Retries reuse the committed charge and snapshot.
+    Request-scoped inputs, overrides and effective rates come from the usage row's versioned payload.
+    Retries reuse the committed charge and snapshot.
     Component names match ``CostBreakdown``: reasoning_cost is part of output_cost, not an additional charge.
     Component sums and float tolerances belong to reconciliation, not this structural contract.
     """

litellm/batches/batch_utils.py

@@ -3,9 +3,10 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from dataclasses import replace as dataclasses_replace
 from enum import Enum
+from itertools import groupby
 from typing import Any, Final, Literal, cast
 
-from pydantic import TypeAdapter
+from pydantic import JsonValue, TypeAdapter
 
 import litellm
 from litellm._logging import verbose_logger
@@ -15,6 +16,7 @@ from litellm.llms.base_llm.ocr.transformation import OCRUsageInfo
 from litellm.llms.bedrock.batches.transformation import titan_embedding_usage_from_batch_output
 from litellm.llms.vertex_ai.batches.transformation import (
     is_native_vertex_batch_output_row,
+    native_vertex_batch_row_model,
     native_vertex_batch_row_stats,
 )
 from litellm.types.llms.openai import Batch
@@ -210,6 +212,7 @@ class _BatchOutputLineStats:
     cache_creation_tokens: int
     reasoning_tokens: int
     model: str | None
+    usage: Usage | None
 
 
 def _classify_output_line_stats(
@@ -268,7 +271,7 @@ def _compute_output_line_stats(
     raw_model: Final = response_body.get("model")
     response_model: Final = raw_model if isinstance(raw_model, str) and raw_model else None
     completion_details: Final = usage.completion_tokens_details
-    line_prompt_cost, line_completion_cost = _output_line_cost(
+    line_prompt_cost, line_completion_cost = _output_line_cost_or_pending(
         response_body=response_body,
         usage=usage,
         custom_llm_provider=custom_llm_provider,
@@ -286,7 +289,15 @@ def _compute_output_line_stats(
         cache_read_tokens=prompt_details["cache_hit_tokens"],
         cache_creation_tokens=prompt_details["cache_creation_tokens"],
         reasoning_tokens=(completion_details.reasoning_tokens if completion_details else None) or 0,
-        model=response_model,
+        model=response_model or model_name,
+        usage=(
+            usage.model_copy(update=ocr_usage.model_dump(exclude_none=True))
+            if (ocr_usage := _ocr_usage_info_from_response_body(response_body)) is not None
+            and ocr_usage.model_dump(exclude_none=True)
+            else usage
+            if response_body.get("usage") is not None or usage.total_tokens > 0
+            else None
+        ),
     )
 
 
@@ -298,6 +309,54 @@ def _ocr_usage_info_from_response_body(response_body: Mapping[str, object]) -> O
     return OCRUsageInfo.model_validate(raw_usage_info)
 
 
+def _output_line_cost_or_pending(
+    response_body: Mapping[str, object],
+    usage: Usage,
+    custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
+    model_name: str | None,
+    response_model: str | None,
+    model_info: ModelInfo | None,
+    vertex_location: str | None,
+) -> tuple[float, float]:
+    try:
+        return _output_line_cost(
+            response_body, usage, custom_llm_provider, model_name, response_model, model_info, vertex_location
+        )
+    except Exception:  # noqa: BLE001  # retain measured usage when existing batch pricing fails
+        verbose_logger.warning("Batch line pricing unavailable; retaining usage with the existing zero-cost fallback")
+        return 0.0, 0.0
+
+
+def _combined_reported_batch_usage(usages: tuple[Usage | None, ...]) -> Mapping[str, object] | None:
+    from litellm.cost_calculator import BaseTokenUsageProcessor
+
+    reported: Final = [usage for usage in usages if usage is not None]
+    if not reported:
+        return None
+    fields: Final = tuple(TypeAdapter(dict[str, JsonValue]).validate_python(usage.model_dump()) for usage in reported)
+    quantities: Final = {
+        key: sum(value for usage in fields if isinstance(value := usage.get(key), (int, float)))
+        for key in OCRUsageInfo.model_fields
+        if any(isinstance(usage.get(key), (int, float)) for usage in fields)
+    }
+    return {**BaseTokenUsageProcessor.combine_usage_objects(usage_objects=reported).model_dump(), **quantities}
+
+
+def _batch_usage_by_request(usages: tuple[tuple[str | None, Usage | None], ...]) -> tuple[Mapping[str, object], ...]:
+    rows: Final = tuple(
+        {"model": model, "usage": usage.model_dump() if usage is not None else None} for model, usage in usages
+    )
+    return tuple(sorted(rows, key=lambda row: json.dumps(row, sort_keys=True)))
+
+
+def _batch_usage_by_model(usages: tuple[tuple[str | None, Usage | None], ...]) -> Mapping[str, object]:
+    ordered: Final = sorted(usages, key=lambda item: item[0] or "")
+    return {
+        model: _combined_reported_batch_usage(tuple(item[1] for item in group))
+        for model, group in groupby(ordered, key=lambda item: item[0] or "")
+    }
+
+
 def _output_line_cost(
     response_body: Mapping[str, object],
     usage: Usage,
@@ -361,6 +420,9 @@ def _aggregate_batch_cost_usage_models(
         completion_tokens=sum(stats.completion_tokens for stats in line_stats),
         reasoning_tokens=sum(stats.reasoning_tokens for stats in line_stats),
         **cache_token_params,
+        batch_usage_by_model=_batch_usage_by_model(tuple((stats.model, stats.usage) for stats in line_stats)),
+        batch_usage_by_request=_batch_usage_by_request(tuple((stats.model, stats.usage) for stats in line_stats)),
+        batch_unknown_usage_requests=sum(stats.usage is None for stats in line_stats),
     )
     batch_models: Final = [model_name] if model_name else [stats.model for stats in line_stats if stats.model]
     total_prompt_cost: Final = sum((stats.prompt_cost for stats in line_stats), 0.0)
@@ -401,17 +463,21 @@ def calculate_vertex_ai_batch_cost_and_usage(
     from litellm.cost_calculator import batch_cost_calculator
     from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig
 
-    row_stats: Final = tuple(
-        native_vertex_batch_row_stats(
-            row,
-            model_name,
-            model_info=model_info,
-            calculate_usage=VertexGeminiConfig.calculate_usage,
-            cost_calculator=batch_cost_calculator,
-            vertex_location=vertex_location,
+    observations: Final = tuple(
+        (
+            native_vertex_batch_row_model(row, model_name),
+            native_vertex_batch_row_stats(
+                row,
+                model_name,
+                model_info=model_info,
+                calculate_usage=VertexGeminiConfig.calculate_usage,
+                cost_calculator=batch_cost_calculator,
+                vertex_location=vertex_location,
+            ),
         )
         for row in vertex_ai_batch_responses
     )
+    row_stats: Final = tuple(stats for _, stats in observations)
     priced: Final = tuple(stats for stats in row_stats if stats is not None)
     total_prompt_cost: Final = sum(stats.prompt_cost for stats in priced)
     total_completion_cost: Final = sum(stats.completion_cost for stats in priced)
@@ -435,6 +501,13 @@ def calculate_vertex_ai_batch_cost_and_usage(
             total_tokens=total_tokens,
             prompt_tokens=prompt_tokens,
             completion_tokens=completion_tokens,
+            batch_usage_by_model=_batch_usage_by_model(
+                tuple((model, stats.usage if stats is not None else None) for model, stats in observations)
+            ),
+            batch_usage_by_request=_batch_usage_by_request(
+                tuple((model, stats.usage if stats is not None else None) for model, stats in observations)
+            ),
+            batch_unknown_usage_requests=sum(stats is None for stats in row_stats),
         ),
         models=(
             [model_name]

litellm/constants.py

@@ -432,6 +432,7 @@ WEBSOCKET_CLOSE_REASON_MAX_BYTES: Final = 123
 DEEPGRAM_DEFAULT_API_BASE: Final = "https://api.deepgram.com/v1"
 NADIR_DEFAULT_API_BASE: Final = "https://api.getnadir.com/v1"
 DEEPGRAM_LISTEN_DEFAULT_MODEL: Final = "nova-3"
+DEEPGRAM_SPEAK_DEFAULT_MODEL: Final = "aura-asteria-en"
 
 BEDROCK_REALTIME_PENDING_OPENING_MESSAGE_SCOPE_KEY: Final = "litellm.bedrock_realtime.pending_opening_message"
 BEDROCK_REALTIME_SESSION_COMMITTED_SCOPE_KEY: Final = "litellm.bedrock_realtime.session_committed"
@@ -675,7 +676,7 @@ request_timeout: float = (
     DEFAULT_REQUEST_TIMEOUT_SECONDS if _request_timeout_from_env is None else _request_timeout_from_env
 )
 request_timeout_explicitly_set: bool = _request_timeout_from_env is not None
-DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0)  # 10 minutes
+DEFAULT_A2A_AGENT_TIMEOUT: Final[float] = get_env_float("DEFAULT_A2A_AGENT_TIMEOUT", 6000.0)
 AGENT_KILL_SWITCH_TIMEOUT_SECONDS: Final = 10.0
 AGENT_KILL_SWITCH_RESPONSE_BODY_MAX_CHARS: Final = 2000
 # Patterns that indicate a localhost/internal URL in A2A agent cards that should be

litellm/cost_calculator.py

@@ -1554,12 +1554,66 @@ def _breakdown_total(litellm_logging_obj: LitellmLoggingObject | None) -> float
     return float(total) if isinstance(total, (int, float)) and not isinstance(total, bool) else None
 
 
+def _cost_map_priced_input_output(
+    model: str | None,
+    custom_llm_provider: str | None,
+    usage: Usage | None,
+    call_type: CallTypesLiteral | None,
+    service_tier: str | None,
+    data_residency: str | None,
+) -> tuple[float, float]:
+    if not model or usage is None:
+        return 0.0, 0.0
+    try:
+        return cost_per_token(
+            model=model,
+            custom_llm_provider=custom_llm_provider,
+            usage_object=usage,
+            call_type=call_type or "completion",
+            service_tier=service_tier,
+            data_residency=data_residency,
+        )
+    except Exception as pricing_error:  # noqa: BLE001  # cost_per_token raises plain Exception for an unmapped model
+        verbose_logger.debug("No cost map price for reported-cost split of %s: %s", model, pricing_error)
+        return 0.0, 0.0
+
+
+def split_reported_cost_by_cost_map_ratio(
+    reported_total: float,
+    model: str | None,
+    custom_llm_provider: str | None,
+    usage: Usage | None,
+    call_type: CallTypesLiteral | None,
+    service_tier: str | None,
+    data_residency: str | None,
+) -> tuple[float, float]:
+    """Apportions a provider-reported total between input and output in the ratio litellm's own cost map would
+    price the same usage, so a per-input-token model books the whole total on input. Unmapped or zero-priced
+    usage keeps the whole total on output."""
+    priced_input, priced_output = _cost_map_priced_input_output(
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        usage=usage,
+        call_type=call_type,
+        service_tier=service_tier,
+        data_residency=data_residency,
+    )
+    priced_total: Final = priced_input + priced_output
+    if priced_total <= 0:
+        return 0.0, reported_total
+    input_cost: Final = reported_total * (priced_input / priced_total)
+    return input_cost, reported_total - input_cost
+
+
 def _adjusted_provider_billed_cost(
     reported_cost: float,
     custom_llm_provider: str | None,
+    pricing_provider: str | None,
     model: str | None,
     region_name: str | None,
     litellm_logging_obj: LitellmLoggingObject | None,
+    usage: Usage | None,
+    call_type: CallTypesLiteral | None,
     service_tier: str | None,
     data_residency: str | None,
 ) -> float:
@@ -1569,10 +1623,19 @@ def _adjusted_provider_billed_cost(
     adjusted_cost, margin_percent, margin_fixed_amount, margin_total_amount = _apply_cost_margin(
         base_cost=discounted_cost, custom_llm_provider=custom_llm_provider
     )
+    input_cost, output_cost = split_reported_cost_by_cost_map_ratio(
+        reported_total=adjusted_cost,
+        model=model,
+        custom_llm_provider=pricing_provider,
+        usage=usage,
+        call_type=call_type,
+        service_tier=service_tier,
+        data_residency=data_residency,
+    )
     _store_cost_breakdown_in_logging_obj(
         litellm_logging_obj=litellm_logging_obj,
-        prompt_tokens_cost_usd_dollar=0.0,
-        completion_tokens_cost_usd_dollar=adjusted_cost,
+        prompt_tokens_cost_usd_dollar=input_cost,
+        completion_tokens_cost_usd_dollar=output_cost,
         cost_for_built_in_tools_cost_usd_dollar=0.0,
         total_cost_usd_dollar=adjusted_cost,
         original_cost=reported_cost,
@@ -2080,9 +2143,16 @@ def completion_cost(
                     return _adjusted_provider_billed_cost(
                         reported_cost=reported_cost,
                         custom_llm_provider=custom_llm_provider,
+                        pricing_provider=_pricing_provider_for_cost_candidate(
+                            candidate_provider=candidate_pricing_provider,
+                            request_provider=request_pricing_provider,
+                            current_provider=custom_llm_provider,
+                        ),
                         model=model_for_discount,
                         region_name=region_name if isinstance(region_name, str) else None,
                         litellm_logging_obj=litellm_logging_obj,
+                        usage=cost_per_token_usage_object,
+                        call_type=call_type,
                         service_tier=service_tier,
                         data_residency=data_residency,
                     )
@@ -2387,10 +2457,28 @@ def response_cost_calculator(
                     provider_response_cost: Final = get_response_cost_from_hidden_params(hidden_params)
                     if provider_response_cost is not None and not custom_pricing:
                         if _breakdown_total(litellm_logging_obj) != provider_response_cost:
+                            header_priced_model: Final = select_model_name_for_cost_calc(
+                                model=model,
+                                completion_response=response_object,
+                                custom_llm_provider=custom_llm_provider,
+                                custom_pricing=custom_pricing,
+                                base_model=base_model,
+                                router_model_id=router_model_id,
+                                region_name=region_name,
+                            )
+                            header_input_cost, header_output_cost = split_reported_cost_by_cost_map_ratio(
+                                reported_total=provider_response_cost,
+                                model=header_priced_model or model,
+                                custom_llm_provider=pricing_provider or custom_llm_provider,
+                                usage=get_usage_object(response_object),
+                                call_type=call_type,
+                                service_tier=service_tier,
+                                data_residency=data_residency,
+                            )
                             _store_cost_breakdown_in_logging_obj(
                                 litellm_logging_obj=litellm_logging_obj,
-                                prompt_tokens_cost_usd_dollar=0.0,
-                                completion_tokens_cost_usd_dollar=provider_response_cost,
+                                prompt_tokens_cost_usd_dollar=header_input_cost,
+                                completion_tokens_cost_usd_dollar=header_output_cost,
                                 cost_for_built_in_tools_cost_usd_dollar=0.0,
                                 total_cost_usd_dollar=provider_response_cost,
                                 original_cost=provider_response_cost,

litellm/endpoints/speech/speech_to_completion_bridge/handler.py

@@ -113,7 +113,7 @@ class SpeechToCompletionBridgeHandler:
             voice=voice,
         )
 
-        with internal_sub_call():
+        with internal_sub_call(logging_obj.litellm_call_id):
             result: Final = completion(
                 **request_data,
             )

litellm/integrations/anthropic_cache_control_hook.py

@@ -13,7 +13,7 @@ import copy
 import itertools
 import os
 import re
-from collections.abc import Callable, Container, Iterable, Mapping, Sequence
+from collections.abc import Callable, Container, Iterable, Iterator, Mapping, Sequence
 from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast
 from urllib.parse import urlparse
 
@@ -445,22 +445,41 @@ def _as_object_list(value: object | None) -> list[object] | None:
     return _validated_object_list(value)
 
 
-def _tool_call_carries_cache_breakpoint(tool_call: object, message: object, answered_tool_ids: Container[str]) -> bool:
+def _server_tool_call_id(tool_call: object) -> str | None:
+    tool_call_id: Final = _attribute_or_key(tool_call, "id")
+    if isinstance(tool_call_id, str) and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX):
+        return tool_call_id
+    return None
+
+
+def _tool_call_carries_cache_breakpoint(tool_call: object, client_tool_call_ids: Container[str]) -> bool:
     if _attribute_or_key(tool_call, "cache_control") is None:
         return False
+    server_tool_call_id: Final = _server_tool_call_id(tool_call)
+    return server_tool_call_id is None or server_tool_call_id in client_tool_call_ids
 
-    return not tool_call_is_rebuilt_as_server_tool_use(
-        _attribute_or_key(tool_call, "id"), _attribute_or_key(message, "provider_specific_fields"), answered_tool_ids
+
+def _server_tool_call_ids_without_a_kept_result(message: object) -> Iterator[str]:
+    provider_specific_fields: Final = _attribute_or_key(message, "provider_specific_fields")
+    tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls")) or ()
+    return (
+        tool_call_id
+        for tool_call in tool_calls
+        if (tool_call_id := _server_tool_call_id(tool_call)) is not None
+        and not tool_call_is_rebuilt_as_server_tool_use(tool_call_id, provider_specific_fields)
     )
 
 
-def _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(
+def _client_tool_call_ids_when_a_server_tool_call_is_marked(
     messages: Sequence[object], model: str | None = None, custom_llm_provider: str | None = None
 ) -> frozenset[str]:
     if not _marks_a_server_tool_call(messages):
         return frozenset()
     if model is not None and not _builds_anthropic_messages(model, custom_llm_provider):
-        return tool_call_ids_answered_in(messages)
+        server_calls_sent_as_client_calls: Final = itertools.chain.from_iterable(
+            map(_server_tool_call_ids_without_a_kept_result, messages)
+        )
+        return tool_call_ids_answered_in(messages) | frozenset(server_calls_sent_as_client_calls)
     return tool_call_ids_answered_by_client(messages, _keeps_mid_conversation_system(model, custom_llm_provider))
 
 
@@ -517,12 +536,7 @@ def _model_info_flags_mid_conversation_system(model: str, custom_llm_provider: s
 
 
 def _is_marked_server_tool_call(tool_call: object) -> bool:
-    tool_call_id: Final = _attribute_or_key(tool_call, "id")
-    return (
-        isinstance(tool_call_id, str)
-        and tool_call_id.startswith(ANTHROPIC_SERVER_TOOL_USE_ID_PREFIX)
-        and _attribute_or_key(tool_call, "cache_control") is not None
-    )
+    return _server_tool_call_id(tool_call) is not None and _attribute_or_key(tool_call, "cache_control") is not None
 
 
 def _tool_carries_cache_breakpoint(tool: object) -> bool:
@@ -668,7 +682,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         reserved_blocks: Final = AnthropicCacheControlHook._blocks_reserved_outside_messages(
             remaining_points, external_breakpoints, openai_dialect
         )
-        answered_tool_ids: Final = _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(
+        answered_tool_ids: Final = _client_tool_call_ids_when_a_server_tool_call_is_marked(
             processed_messages, model, custom_llm_provider
         )
         breakpoints_before: Final = AnthropicCacheControlHook.count_request_cache_breakpoints(
@@ -757,7 +771,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         )
         message_list: Final = tuple(messages)
         client_answers: Final = (
-            _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(message_list)
+            _client_tool_call_ids_when_a_server_tool_call_is_marked(message_list)
             if answered_tool_ids is None
             else answered_tool_ids
         )
@@ -849,7 +863,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         so earlier points win when slots are scarce.
         """
         client_answers: Final = (
-            _tool_call_ids_answered_by_client_when_a_server_tool_call_is_marked(messages)
+            _client_tool_call_ids_when_a_server_tool_call_is_marked(messages)
             if answered_tool_ids is None
             else answered_tool_ids
         )
@@ -939,11 +953,7 @@ class AnthropicCacheControlHook(CustomPromptManagement):
         content_count: Final = sum(1 for block in content if _carries_cache_breakpoint(block)) if content else 0
         tool_calls: Final = _as_object_list(_attribute_or_key(message, "tool_calls"))
         tool_call_count: Final = (
-            sum(
-                1
-                for tool_call in tool_calls
-                if _tool_call_carries_cache_breakpoint(tool_call, message, answered_tool_ids)
-            )
+            sum(1 for tool_call in tool_calls if _tool_call_carries_cache_breakpoint(tool_call, answered_tool_ids))
             if tool_calls
             else 0
         )

litellm/litellm_core_utils/core_helpers.py

@@ -1074,9 +1074,14 @@ def billed_usage_cost(response: object, custom_llm_provider: str | None) -> floa
 _MESSAGES_REPORTED_COST_PROVIDERS: Final = frozenset({"opencode_go"})
 
 
+def reports_native_messages_cost(custom_llm_provider: object) -> bool:
+    """Whether a Messages upstream is one whose own reported cost litellm bills from instead of the price map."""
+    return isinstance(custom_llm_provider, str) and custom_llm_provider in _MESSAGES_REPORTED_COST_PROVIDERS
+
+
 def native_messages_reported_cost(usage: object, custom_llm_provider: object) -> float | None:
     """Only a Messages upstream known to price its own responses is trusted, every other one keeps the price map."""
-    if not isinstance(custom_llm_provider, str) or custom_llm_provider not in _MESSAGES_REPORTED_COST_PROVIDERS:
+    if not isinstance(custom_llm_provider, str) or not reports_native_messages_cost(custom_llm_provider):
         return None
     return provider_reported_usage_cost(usage, custom_llm_provider)
 

litellm/litellm_core_utils/litellm_logging.py

@@ -16,9 +16,10 @@ import traceback
 from collections.abc import Awaitable, Callable, Coroutine, Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass
 from datetime import datetime as dt_object
+from enum import Enum
 from functools import lru_cache
 from types import MappingProxyType, TracebackType
-from typing import TYPE_CHECKING, Any, Final, Literal, NamedTuple, cast
+from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, NamedTuple, cast
 
 from httpx import Response, Timeout
 from pydantic import BaseModel, JsonValue, TypeAdapter
@@ -47,6 +48,7 @@ from litellm.constants import (
     KUBERNETES_POD_ROUTING_KEY,
     LITELLM_LOGGING_NO_UPSTREAM_LLM_CALL,
     PROVIDER_REQUEST_ID_HEADERS,
+    REALTIME_METERED_USAGE_KEY,
     REDACTED_BY_LITELLM,
 )
 from litellm.cost_calculator import (
@@ -84,6 +86,7 @@ from litellm.litellm_core_utils.core_helpers import (
     proxy_stamped_kubernetes_pod_routing,
     proxy_stamped_used_client_oauth_token,
     reconstruct_model_name,
+    reports_native_messages_cost,
     set_response_cost_in_hidden_params,
     single_log_line,
 )
@@ -163,6 +166,7 @@ from litellm.types.llms.openai import (
     ResponseIncompleteEvent,
     ResponsesAPIResponse,
 )
+from litellm.types.llms.vertex_ai import GenerateContentResponseBody, UsageMetadata
 from litellm.types.mcp import MCPPostCallResponseObject
 from litellm.types.prompts.init_prompts import PromptSpec
 from litellm.types.rerank import RerankResponse, get_rerank_usage
@@ -266,6 +270,9 @@ if TYPE_CHECKING:
     from litellm.litellm_core_utils.llm_cost_calc.utils import BilledTokenRates
     from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector
     from litellm.proxy.hooks.autorouter_baseline_cache import BaselineCacheContext, CapturedBaselineObservation
+    from litellm.proxy.hooks.search_free_tier import SearchAllowanceAllocation
+    from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
+    from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
 try:
     from litellm_enterprise.enterprise_callbacks.callback_controls import (
         EnterpriseCallbackControls,
@@ -571,6 +578,7 @@ def _resolve_mantle_region_for_cost(
 _PER_CALL_LITELLM_PARAM_DICTS: Final = frozenset({"metadata", "litellm_metadata"})
 _LITELLM_PARAMS: Final = TypeAdapter(dict[str, object])
 _SERVED_ANTHROPIC_BODY: Final = TypeAdapter(dict[str, object])
+_GOOGLE_USAGE_FIELDS: Final = frozenset(UsageMetadata.__annotations__)
 _REDACTED_RESULT: Final = TypeAdapter(object)
 _GUARDRAIL_INFORMATION_KEY: Final = "standard_logging_guardrail_information"
 
@@ -630,7 +638,60 @@ def same_attempt_marker(first: object, second: object) -> bool:
     return first is second or first == second
 
 
+class UsageObservation(Enum):
+    KNOWN = "known"
+    UNKNOWN = "unknown"
+    ESTIMATED = "estimated"
+
+
+@dataclass(frozen=True, slots=True)
+class UsageCaptureObservation:
+    usage_json: str | None
+    provenance: UsageObservation
+    response_cost: float | None = None
+
+    @classmethod
+    def from_usage(
+        cls, usage: Usage | None, provenance: UsageObservation, response_cost: float | None = None
+    ) -> "UsageCaptureObservation | UsageCaptureError":
+        from litellm.proxy.spend_tracking.usage_meter import UsageCaptureError
+
+        try:
+            serialized: Final = (
+                None if usage is None or provenance is UsageObservation.UNKNOWN else usage.model_dump_json()
+            )
+        except Exception as error:  # noqa: BLE001  # observation failures must not interrupt legacy retry handling
+            return UsageCaptureError(str(error))
+        else:
+            return cls(serialized, provenance, response_cost)
+
+
 class Logging(LiteLLMLoggingBaseClass):
+    usage_capture_callback: ClassVar[
+        Callable[
+            ["Logging", object, datetime.datetime, datetime.datetime, bool], "CapturedUsage | UsageCaptureError | None"
+        ]
+        | None
+    ] = None
+
+    def capture_usage_before_logging(
+        self,
+        response: object,
+        start_time: datetime.datetime | None = None,
+        end_time: datetime.datetime | None = None,
+        *,
+        freeze_only: bool = False,
+    ) -> None:
+        capture: Final = type(self).usage_capture_callback
+        if capture is not None and (not self.stream or self._is_assembled_stream_success(response)):
+            capture(
+                self,
+                response,
+                start_time or self.start_time,
+                end_time or datetime.datetime.now(datetime.timezone.utc),
+                freeze_only,
+            )
+
     global \
         supabaseClient, \
         promptLayerLogger, \
@@ -828,6 +889,7 @@ class Logging(LiteLLMLoggingBaseClass):
         self.finalized_cost: FinalizedCost | None = None
         self.prepared_cost: PreparedCost | None = None
         self.cost_allowance_discount: float | None = None
+        self.cost_allowance_allocation: SearchAllowanceAllocation | None = None
         self.cost_finalization_lock = asyncio.Lock()
         self.deferred_success_result: tuple[object, datetime.datetime, datetime.datetime] | None = None
         self.billed_token_rates: BilledTokenRates | None = None
@@ -884,7 +946,7 @@ class Logging(LiteLLMLoggingBaseClass):
         new_call.start_time = start_time
         new_call.litellm_params = litellm_params
         new_call.model_call_details = {
-            **self.model_call_details,
+            **{key: value for key, value in self.model_call_details.items() if not key.startswith("_accounting_")},
             "litellm_call_id": litellm_call_id,
             "start_time": start_time,
             "first_api_call_start_time": None,
@@ -894,6 +956,7 @@ class Logging(LiteLLMLoggingBaseClass):
         new_call.finalized_cost = None
         new_call.prepared_cost = None
         new_call.cost_allowance_discount = None
+        new_call.cost_allowance_allocation = None
         new_call.cost_finalization_lock = asyncio.Lock()
         new_call.deferred_success_result = None
         new_call.streaming_chunks = []
@@ -2424,7 +2487,7 @@ class Logging(LiteLLMLoggingBaseClass):
 
     _is_sync_litellm_request = is_sync_litellm_request
 
-    def _is_assembled_stream_success(self, result=None) -> bool:
+    def _is_assembled_stream_success(self, result: object = None) -> bool:
         """Final assembled stream export (not a per-chunk success call).
 
         Per-chunk callers pass a ``ModelResponseStream`` (or ``None``); the
@@ -2459,7 +2522,7 @@ class Logging(LiteLLMLoggingBaseClass):
         """
         from litellm.litellm_core_utils.thread_pool_executor import executor
 
-        if self._is_assembled_stream_success(result):
+        if self._is_assembled_stream_success(TypeAdapter(object).validate_python(result)):
             if self.model_call_details.get("has_dispatched_final_stream_success"):
                 return
             self.model_call_details["has_dispatched_final_stream_success"] = True
@@ -2477,6 +2540,11 @@ class Logging(LiteLLMLoggingBaseClass):
             )
             return
 
+        self.capture_usage_before_logging(
+            TypeAdapter(object).validate_python(result),
+            TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(start_time),
+            TypeAdapter[datetime.datetime | None](datetime.datetime | None).validate_python(end_time),
+        )
         await self.async_success_handler(
             result,
             start_time=start_time,
@@ -2503,6 +2571,17 @@ class Logging(LiteLLMLoggingBaseClass):
             return self.prepared_cost
         recorded_cache_hit: Final[object] = self.model_call_details.get("cache_hit")
         deferred: Final = self.deferred_success_result
+        observed_response: Final = _LITELLM_PARAMS.validate_python(
+            _extract_response_obj_and_hidden_params(deferred[0] if deferred else response, None)[0]
+        )
+        self.model_call_details["_accounting_usage_observed"] = (
+            UsageObservation.KNOWN
+            if (
+                self.model_call_details.get("combined_usage_object")
+                or StandardLoggingPayloadSetup.get_usage_as_dict(observed_response)
+            )
+            else UsageObservation.UNKNOWN
+        )
         prepared: Final = self.prepare_success_logging(
             result=deferred[0] if deferred is not None else response,
             start_time=deferred[1] if deferred is not None else self.start_time,
@@ -2540,17 +2619,29 @@ class Logging(LiteLLMLoggingBaseClass):
             },
             usage,
         )
-        self.record_partial_usage_for_failure(usage, raw_cost)
+        self.record_partial_usage_for_failure(
+            usage, raw_cost, observation=self.model_call_details["_accounting_usage_observed"]
+        )
         return self.prepared_cost
 
     def record_finalized_cost(self, finalized: FinalizedCost) -> None:
         self.finalized_cost = finalized
         self.model_call_details["response_cost"] = finalized.amount
 
-    def record_partial_usage_for_failure(self, usage: Usage, response_cost: float) -> None:
+    def record_usage_capture(self, captured: "CapturedUsage") -> None:
+        self.model_call_details["_accounting_capture"] = captured
+
+    def record_usage_pricing(self, pricing: object) -> None:
+        self.model_call_details["_accounting_pricing_snapshot"] = pricing
+
+    def record_partial_usage_for_failure(
+        self, usage: Usage, response_cost: float, *, observation: UsageObservation = UsageObservation.KNOWN
+    ) -> None:
         """Stash what an interrupted stream already consumed so the failure log bills it instead of zero."""
         self.model_call_details["combined_usage_object"] = usage
         self.model_call_details["response_cost"] = response_cost
+        if type(self).usage_capture_callback is not None:
+            self.model_call_details["_accounting_usage_observed"] = observation
 
     def record_failure_after_earlier_rounds(
         self, earlier: BilledRoundsSpend, failed_usage: Usage, failed_cost: float
@@ -2630,6 +2721,11 @@ class Logging(LiteLLMLoggingBaseClass):
         own usage while the request's failure log still bills every attempt."""
         usage: Final = self._model_call_details_view.get("combined_usage_object")
         response_cost: Final = self._model_call_details_view.get("response_cost")
+        if isinstance(usage, Usage) and type(self).usage_capture_callback is not None:
+            self.capture_failed_attempt_usage(usage)
+        self.model_call_details.pop("_accounting_capture", None)
+        self.model_call_details.pop("_accounting_pricing_snapshot", None)
+        self.model_call_details.pop("_accounting_usage_observed", None)
         self.model_call_details.pop("combined_usage_object", None)
         self.model_call_details.pop("response_cost", None)
         if isinstance(usage, Usage):
@@ -2639,6 +2735,53 @@ class Logging(LiteLLMLoggingBaseClass):
                 (usage, float(response_cost) if isinstance(response_cost, (int, float)) else 0.0),
             )
 
+    def capture_failed_attempt_usage(self, usage: Usage) -> None:
+        from litellm.constants import BILLED_RETRY_ATTEMPT_METADATA_KEY
+        from litellm.proxy.spend_tracking.usage_capture import CapturedUsage
+
+        capture: Final = type(self).usage_capture_callback
+        if capture is None:
+            return
+        attempt: Final = self.attempt_scoped_copy()
+        attempt.prepared_cost = None
+        attempt.finalized_cost = None
+        identity: Final = (
+            f"{self.litellm_call_id}-attempt-{len(self.model_call_details.get('failed_attempts_usage', ()))}"
+        )
+        params: Final = _LITELLM_PARAMS.validate_python(self.model_call_details.get("litellm_params") or {})
+        metadata: Final = StandardLoggingPayloadSetup.merge_litellm_metadata(params)
+        observed: Final[object] = self.model_call_details.get("_accounting_usage_observed")
+        cost: Final[object] = self.model_call_details.get("response_cost")
+        attempt.model_call_details = {
+            **attempt.model_call_details,
+            "litellm_call_id": identity,
+            "parent_request_id": self.litellm_call_id,
+            "_accounting_capture": None,
+            "_accounting_prior_captures": (),
+            "_accounting_usage_snapshot": UsageCaptureObservation.from_usage(
+                usage,
+                observed if isinstance(observed, UsageObservation) else UsageObservation.KNOWN,
+                float(cost) if isinstance(cost, (int, float)) else None,
+            ),
+            "litellm_params": {
+                **params,
+                "metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
+                "litellm_metadata": {**metadata, BILLED_RETRY_ATTEMPT_METADATA_KEY: True, "status": "failure"},
+            },
+        }
+        captured: Final = capture(
+            attempt,
+            {"id": identity, "usage": usage.model_dump(), "status": "failed"},
+            self.start_time,
+            datetime.datetime.now(datetime.timezone.utc),
+            False,
+        )
+        if isinstance(captured, CapturedUsage):
+            self.model_call_details["_accounting_prior_captures"] = (
+                *self.model_call_details.get("_accounting_prior_captures", ()),
+                captured,
+            )
+
     def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
         """Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
         self.model_call_details["continued_partial_usage"] = usage
@@ -2796,6 +2939,55 @@ class Logging(LiteLLMLoggingBaseClass):
 
     _update_completion_start_time = update_completion_start_time
 
+    def normalize_endpoint_logging_result(self, result: object) -> object:
+        call_type: Final[object] = self.call_type
+        if call_type == CallTypes.anthropic_messages.value:
+            return self._anthropic_messages_logged_response(result=result)
+        if call_type in (CallTypes.generate_content.value, CallTypes.agenerate_content.value):
+            return self._handle_non_streaming_google_genai_generate_content_response_logging(result=result)
+        if call_type in (CallTypes.asend_message.value, CallTypes.send_message.value):
+            return self._handle_a2a_response_logging(result=result)
+        return result
+
+    def usage_capture_response(self, result: object) -> object:
+        if self.stream and not self._is_assembled_stream_success(result):
+            return None
+        copied: Final = result.model_copy(deep=True) if isinstance(result, BaseModel) else result
+        normalized: Final = self.normalize_logging_result(self.normalize_endpoint_logging_result(copied))
+        if self.model_call_details.get("call_type") == CallTypes.arealtime.value and isinstance(result, list):
+            frames: Final = TypeAdapter(tuple[dict[str, object], ...]).validate_python(result)
+            measured: Final = any(
+                isinstance(_LITELLM_PARAMS.validate_python(frame.get("response") or {}).get("usage"), Mapping)
+                for frame in frames
+                if frame.get("type") == "response.done"
+            )
+            if not measured:
+                metered: Final = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
+                return {
+                    "id": self.litellm_call_id,
+                    "usage": {"metered_usage": TypeAdapter(object).dump_python(metered, mode="json")}
+                    if metered is not None
+                    else None,
+                }
+        assembled: Final = self._get_assembled_streaming_response(
+            normalized, self.start_time, self.start_time, True, []
+        )
+        return normalized if assembled is None else assembled
+
+    def usage_capture_pricing(self, result: object) -> Mapping[str, JsonValue]:
+        from litellm.utils import count_characters
+
+        return TypeAdapter(dict[str, JsonValue]).validate_python(
+            {
+                "custom_pricing": self.custom_pricing_for(result),
+                "custom_cost_per_token": self.custom_cost_per_token,
+                "custom_cost_per_second": self.custom_cost_per_second,
+                "prompt_characters": count_characters(self._prompt_for_cost_calculation())
+                if self.model_call_details.get("call_type") in (CallTypes.speech.value, CallTypes.aspeech.value)
+                else None,
+            }
+        )
+
     def normalize_logging_result(self, result: object) -> object:
         """
         Some endpoints return a different type of result than what is expected by the logging system.
@@ -2806,8 +2998,20 @@ class Logging(LiteLLMLoggingBaseClass):
             combined_usage_object: Final = (
                 RealtimeAPITokenUsageProcessor.collect_and_combine_usage_from_realtime_stream_results(results=result)
             )
+            metered: Final[object] = self.model_call_details.get(REALTIME_METERED_USAGE_KEY)
+            captured_usage: Final = (
+                combined_usage_object.model_copy(
+                    update={
+                        "metered_usage": TypeAdapter(dict[str, JsonValue]).validate_python(
+                            TypeAdapter(object).dump_python(metered, mode="json")
+                        )
+                    }
+                )
+                if metered is not None
+                else combined_usage_object
+            )
             logging_result = RealtimeAPITokenUsageProcessor.create_logging_realtime_object(
-                usage=combined_usage_object,
+                usage=captured_usage,
                 results=result,
             )
 
@@ -3068,7 +3272,14 @@ class Logging(LiteLLMLoggingBaseClass):
                 start_time = self.start_time
             if end_time is None:
                 end_time = datetime.datetime.now()
-            if self.completion_start_time is None:
+            # A request re-dispatched to a fallback reuses this logging object but runs in
+            # its own window, so a first-token time from the previous attempt cannot belong
+            # to it. Keeping it makes the attempt report a negative TTFT.
+            first_token_seconds: Final = _timestamp_seconds(self.completion_start_time)
+            attempt_start_seconds: Final = _timestamp_seconds(start_time)
+            if first_token_seconds is None or (
+                attempt_start_seconds is not None and first_token_seconds < attempt_start_seconds
+            ):
                 self.completion_start_time = end_time
                 self.model_call_details["completion_start_time"] = self.completion_start_time
 
@@ -3080,15 +3291,7 @@ class Logging(LiteLLMLoggingBaseClass):
                 else cache_hit
             )
 
-            if self.call_type == CallTypes.anthropic_messages.value:
-                result = self._anthropic_messages_logged_response(result=result)
-            elif (
-                self.call_type == CallTypes.generate_content.value
-                or self.call_type == CallTypes.agenerate_content.value
-            ):
-                result = self._handle_non_streaming_google_genai_generate_content_response_logging(result=result)
-            elif self.call_type == CallTypes.asend_message.value or self.call_type == CallTypes.send_message.value:
-                result = self._handle_a2a_response_logging(result=result)
+            result = self.normalize_endpoint_logging_result(result)
 
             logging_result: Final = self.normalize_logging_result(result=result)
             if cache_hit is None and _served_from_cache_at_no_charge(logging_result):
@@ -3441,6 +3644,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     self._with_client_facing_stream_model(complete_streaming_response)
                 )
                 self.model_call_details["complete_streaming_response"] = logged_streaming_response
+                self.capture_usage_before_logging(logged_streaming_response, start_time, end_time)
                 ## STANDARDIZED LOGGING PAYLOAD
                 if not for_async_request:
                     self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
@@ -3964,6 +4168,8 @@ class Logging(LiteLLMLoggingBaseClass):
                     cost_for_built_in_tools_cost_usd_dollar=0.0,
                 )
 
+        self.capture_usage_before_logging(result, start_time, end_time)
+
         self.truncated_messages_for_logging = await truncate_base64_in_messages_async(
             StandardLoggingPayloadSetup.append_system_prompt_messages(
                 kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
@@ -4990,16 +5196,34 @@ class Logging(LiteLLMLoggingBaseClass):
 
     def billable_result(self, result: object) -> object:
         """The response the cost calculator prices: a /v1/messages body served through the chat
-        bridge is replaced by the ModelResponse its spend row is built from, anything else is
+        bridge is replaced by the ModelResponse its spend row is built from, a native one is only
+        converted when its provider reports the cost the bill is read from, anything else is
         priced as is."""
         served: Final[object] = result
         call_type: Final[object] = getattr(self, "call_type", None)
         if not isinstance(result, dict):
             return served
-        if self.bridged_model_response is None and call_type != CallTypes.anthropic_messages.value:
+        if self.bridged_model_response is not None:
+            return self._anthropic_messages_logged_response(result=served)
+        if call_type != CallTypes.anthropic_messages.value:
             return served
+        if not reports_native_messages_cost(self.model_call_details.get("custom_llm_provider")):
+            return self._served_with_requested_speed(result=served)
         return self._anthropic_messages_logged_response(result=served)
 
+    def _served_with_requested_speed(self, result: object) -> object:
+        """A native /v1/messages body priced as served bills the speed the client asked for when its
+        usage block reports none, the same speed its spend row is priced at."""
+        body: Final = _SERVED_ANTHROPIC_BODY.validate_python(result)
+        usage: Final = body.get("usage")
+        requested: Final = _LITELLM_PARAMS.validate_python(getattr(self, "optional_params", None) or {}).get("speed")
+        if not isinstance(requested, str) or not isinstance(usage, dict):
+            return result
+        reported: Final = _SERVED_ANTHROPIC_BODY.validate_python(usage)
+        if isinstance(reported.get("speed"), str):
+            return result
+        return {**body, "usage": {**reported, "speed": requested}}
+
     def _anthropic_messages_logged_response(self, result: object) -> ModelResponse:
         """
         The ModelResponse a /v1/messages spend_logs row is built from.
@@ -6808,45 +7032,19 @@ class StandardLoggingPayloadSetup:
 
     @staticmethod
     def get_usage_from_response_obj(response_obj: dict | None, combined_usage_object: Usage | None = None) -> Usage:
-        ## BASE CASE ##
         if combined_usage_object is not None:
             return combined_usage_object
-        if response_obj is None:
-            return Usage(
-                prompt_tokens=0,
-                completion_tokens=0,
-                total_tokens=0,
-            )
-
-        usage: Final = response_obj.get("usage", None)
-        if usage is None:
-            rerank_usage: Final = get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj))
-            if rerank_usage is not None:
-                return Usage(**rerank_usage)
-        if usage is None or (not isinstance(usage, dict) and not isinstance(usage, Usage)):
-            return Usage(
-                prompt_tokens=0,
-                completion_tokens=0,
-                total_tokens=0,
-            )
-        elif isinstance(usage, Usage):
-            return usage
-        elif isinstance(usage, ResponseAPIUsage):
-            return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage)
-        elif isinstance(usage, dict):
-            if ResponseAPILoggingUtils.is_response_api_usage(usage):
-                return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(usage)
-            if InteractionsUsageObjectTransformation.is_interactions_usage_object(usage):
-                return InteractionsUsageObjectTransformation.transform_interactions_usage_object(usage)
-            return Usage(**usage)
-
-        raise ValueError(f"usage is required, got={usage} of type {type(usage)}")
+        reported: Final = response_obj.get("usage") if response_obj else None
+        if isinstance(reported, Usage):
+            return reported
+        normalized: Final = StandardLoggingPayloadSetup.get_usage_as_dict(response_obj)
+        return Usage(**normalized) if normalized else Usage(prompt_tokens=0, completion_tokens=0, total_tokens=0)
 
     @staticmethod
     def get_usage_as_dict(
-        response_obj: dict | None,
+        response_obj: dict[str, Any] | None,
         combined_usage_object: Usage | None = None,
-    ) -> dict:
+    ) -> dict[str, Any]:
         """
         Like get_usage_from_response_obj but returns a plain dict, skipping
         the Pydantic Usage construction on the hot path.
@@ -6858,7 +7056,21 @@ class StandardLoggingPayloadSetup:
             return _empty
         _raw: Final = response_obj.get("usage", None)
         if _raw is None:
-            return dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty)
+            google_usage: Final[object] = response_obj.get("usageMetadata")
+            if isinstance(google_usage, dict) and _GOOGLE_USAGE_FIELDS.intersection(google_usage):
+                return litellm.VertexGeminiConfig.calculate_usage(
+                    cast(  # cast-ok: retain converter-supported modalities and token_count aliases
+                        GenerateContentResponseBody, response_obj
+                    )
+                ).model_dump()
+            reported: Final = response_obj.get("usage_info")
+            if isinstance(reported, BaseModel):
+                return reported.model_dump(exclude_none=True)
+            return (
+                {key: value for key, value in reported.items() if value is not None}
+                if isinstance(reported, dict)
+                else dict(get_rerank_usage(_RESPONSE_MAPPING.validate_python(response_obj)) or _empty)
+            )
         if isinstance(_raw, ResponseAPIUsage):
             return ResponseAPILoggingUtils.transform_response_api_usage_to_chat_usage(_raw).model_dump()
         if isinstance(_raw, dict):

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -238,7 +238,7 @@ def is_non_content_values_set(message: AllMessageValues) -> bool:
     return any(message.get(key, None) is not None for key in message if key not in ignore_keys)
 
 
-_IMAGE_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_image", "image"})
+_IMAGE_CONTENT_PART_TYPES: Final = ("image_url", "input_image", "image", "computer_screenshot")
 _IMAGE_SCAN_MAX_DEPTH: Final = 4
 
 
@@ -557,15 +557,24 @@ def as_openai_image_part(part: Mapping[str, object]) -> ChatCompletionImageObjec
     return {"type": "image_url", "image_url": {"url": url}} if url else None
 
 
-def request_contains_image_content(messages: Sequence[Mapping[str, object]]) -> bool:
-    """Whether any message carries an image content part, across the dialects that reach
-    pre-routing hooks untranslated: chat-completions ``image_url``, Responses ``input_image``,
-    and Anthropic Messages ``image``, including images nested inside ``tool_result`` blocks."""
-    return any(
-        isinstance(content, list) and _content_parts_contain_image(content)
-        for message in messages
-        for content in (message.get("content"),)
-    )
+def _message_contains_image_content(message: object) -> bool:
+    match message:
+        case {"content": list()}:
+            return _content_parts_contain_image(content_parts(_CONTENT_PART.validate_python(message)))
+        case {"type": "function_call_output" | "custom_tool_call_output" | "tool_result", "output": list()}:
+            output: Final = _CONTENT_PART.validate_python(message)["output"]
+            return _content_parts_contain_image(_CONTENT_PARTS.validate_python(output))
+        case {"type": "computer_call_output", "output": dict()}:
+            return _content_parts_contain_image((_CONTENT_PART.validate_python(message)["output"],))
+        case _:
+            return False
+
+
+def request_contains_image_content(messages: object) -> bool:
+    """Inspect native message content without translating or validating the request."""
+    if not isinstance(messages, (list, tuple)):
+        return False
+    return any(_message_contains_image_content(message) for message in _CONTENT_PARTS.validate_python(messages))
 
 
 def audio_or_image_in_message_content(message: AllMessageValues) -> bool:

litellm/litellm_core_utils/realtime_streaming.py

@@ -631,6 +631,8 @@ class RealTimeStreaming:
             if self.session_tools or self.tool_calls:
                 self.logging_obj.model_call_details["realtime_tools"] = self.session_tools
                 self.logging_obj.model_call_details["realtime_tool_calls"] = self.tool_calls
+            if LiteLLMLogging.usage_capture_callback is not None:
+                self.logging_obj.capture_usage_before_logging(self.messages)
             # Route through the bounded logging worker (per-coroutine timeout +
             # concurrency cap) instead of a bare create_task, so a slow callback
             # can't leave suspended tasks pinning each call's response in memory.

litellm/litellm_core_utils/streaming_handler.py

@@ -349,11 +349,14 @@ class CustomStreamWrapper:
         make_call: Callable | None = None,
         _response_headers: dict | httpx.Headers | None = None,
         count_prompt_tokens: Callable[[], int] | None = None,
+        is_mock: bool = False,
     ):
         self.model = model
         self.make_call = make_call
         self.count_prompt_tokens = count_prompt_tokens
         self.custom_llm_provider = custom_llm_provider
+        self.is_mock = is_mock
+        self.uses_openai_chat_stream = False
         self.logging_obj: LiteLLMLoggingObject = logging_obj
         self.completion_stream = completion_stream
         self.sent_first_chunk = False
@@ -376,6 +379,7 @@ class CustomStreamWrapper:
         self._upstream_provider: str | None = None
         self.received_finish_reason: str | None = None
         self.intermittent_finish_reason: str | None = None  # finish reasons that show up mid-stream
+        self.received_any_finish_reason = False  # a later chunk can reset intermittent_finish_reason
         self.special_tokens = [
             "<|assistant|>",
             "<|system|>",
@@ -1302,8 +1306,8 @@ class CustomStreamWrapper:
             if _is_delta_empty:
                 model_response.choices[0].delta = Delta(content=self._close_open_think_block())
                 # get any function call arguments
-                model_response.choices[0].finish_reason = map_finish_reason(
-                    finish_reason=self.received_finish_reason
+                model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
+                    map_finish_reason(finish_reason=self.received_finish_reason)
                 )  # ensure consistent output to openai
 
                 self.sent_last_chunk = True
@@ -1597,6 +1601,7 @@ class CustomStreamWrapper:
             if response_obj["is_finished"]:
                 self.received_finish_reason = response_obj["finish_reason"]
         else:  # openai / azure chat model
+            self.uses_openai_chat_stream = True
             if isinstance(chunk, ChatCompletionChunk):
                 self._openai_handler_receives_chat_chunks = True
                 if any(str(choice.finish_reason) == "abort" for choice in chunk.choices or ()):
@@ -1617,6 +1622,7 @@ class CustomStreamWrapper:
                 return _ProviderChunkEarlyReturn(None)
             completion_obj["content"] = response_obj["text"]
             self.intermittent_finish_reason = response_obj.get("finish_reason", None)
+            self.received_any_finish_reason |= response_obj["finish_reason"] is not None
             if response_obj["is_finished"]:
                 if response_obj["finish_reason"] == "error":
                     raise Exception(
@@ -1737,6 +1743,8 @@ class CustomStreamWrapper:
                                         # if function returned but type set to None - mistral's api returns type: None
                                         tool["type"] = "function"
                             model_response.choices[0].delta = Delta(**_json_delta)
+                            if model_response.choices[0].delta.tool_calls:
+                                self.tool_call = True
                         except Exception as e:
                             verbose_logger.exception(
                                 "litellm.CustomStreamWrapper.chunk_creator(): Exception occured - %s", e
@@ -1966,6 +1974,11 @@ class CustomStreamWrapper:
         if self.logging_obj.is_sync_litellm_request(litellm_params):
             self.logging_obj.success_handler(processed_chunk, None, None, cache_hit)
 
+    def _normalize_tool_call_finish_reason(self, finish_reason: str) -> str:
+        if finish_reason == "stop" and self.tool_call:
+            return "tool_calls"
+        return finish_reason
+
     def finish_reason_handler(self):
         model_response: Final = self.model_response_creator(
             hidden_params=None if self.received_model_id is None else {"received_model_id": self.received_model_id}
@@ -1981,21 +1994,30 @@ class CustomStreamWrapper:
         if served_tier is not None:
             setattr(model_response, "service_tier", served_tier)  # noqa: B010  # pydantic extra, not a declared field
         _finish_reason: Final = self.received_finish_reason or self.intermittent_finish_reason
-        if _finish_reason is not None:
-            model_response.choices[0].finish_reason = _finish_reason
-        else:
-            model_response.choices[0].finish_reason = "stop"
-
-        ## if tool use
-        if (
-            model_response.choices[0].finish_reason == "stop" and self.tool_call
-        ):  # don't overwrite for other - potential error finish reasons
-            model_response.choices[0].finish_reason = "tool_calls"
+        model_response.choices[0].finish_reason = self._normalize_tool_call_finish_reason(
+            _finish_reason if _finish_reason is not None else "stop"
+        )
         close_tag: Final = self._close_open_think_block()
         if close_tag is not None:
             model_response.choices[0].delta.content = close_tag
         return model_response
 
+    def _should_fail_on_clean_eof(self) -> bool:
+        return (
+            not self.is_mock
+            and self.uses_openai_chat_stream
+            and self.received_finish_reason is None
+            and self.intermittent_finish_reason is None
+            and not self.received_any_finish_reason
+        )
+
+    @staticmethod
+    def _stream_eof_error() -> httpx.RemoteProtocolError:
+        return httpx.RemoteProtocolError(
+            "Stream ended before a finish_reason was received",
+            request=httpx.Request(method="POST", url="https://docs.litellm.ai/docs"),
+        )
+
     def _final_chunk_for_logging(self, processed_chunk: "ModelResponseStream") -> "ModelResponseStream | None":
         if self.received_finish_reason is not None or self.intermittent_finish_reason is not None:
             return processed_chunk
@@ -2181,6 +2203,13 @@ class CustomStreamWrapper:
                     return response
 
         except StopIteration:
+            if self._should_fail_on_clean_eof():
+                error: Final = self._stream_eof_error()
+                eof_traceback: Final = traceback.format_exc()
+                self._record_partial_usage_for_failure()
+                threading.Thread(target=self.logging_obj.failure_handler, args=(error, eof_traceback)).start()
+                self.close()
+                self._handle_stream_fallback_error(error)
             if self.sent_last_chunk is True:
                 complete_streaming_response: Final = self._build_complete_streaming_response()
                 response = self._new_usage_chunk()
@@ -2384,6 +2413,9 @@ class CustomStreamWrapper:
                             continue
                         return usage_free_chunk
         except (StopAsyncIteration, StopIteration):
+            if self._should_fail_on_clean_eof():
+                await self.aclose()
+                self._log_stream_failure_and_raise(self._stream_eof_error())
             return await self._finalize_completed_stream(cache_hit=cache_hit)
         except httpx.TimeoutException as e:  # if httpx read timeout error occues
             traceback_exception = traceback.format_exc()

litellm/llms/a2a/common_utils.py

@@ -160,8 +160,11 @@ async def resolve_a2a_hop_auth_header(
     litellm_params: Mapping[str, object],
     custom_llm_provider: object,
     resolve_entra_header: AgentAuthHeaderResolver = resolve_azure_ai_agent_auth_header,
+    agent_extra_headers: Mapping[str, str] | None = None,
 ) -> Mapping[str, str] | None:
-    """Entra credentials authenticate the A2A hop only; a completion-bridge agent hands them to the model provider it bridges to."""
-    if not a2a_hop_uses_entra(litellm_params, custom_llm_provider):
+    if a2a_hop_uses_entra(litellm_params, custom_llm_provider):
+        return await resolve_entra_header(litellm_params)
+    if custom_llm_provider or any(name.lower() == "authorization" for name in (agent_extra_headers or {})):
         return None
-    return await resolve_entra_header(litellm_params)
+    api_key: Final = litellm_params.get("api_key")
+    return {"Authorization": f"Bearer {api_key}"} if isinstance(api_key, str) and api_key else None

litellm/llms/anthropic/messages/adapters/handler.py

@@ -118,9 +118,10 @@ def _extract_proxy_litellm_metadata(
 
 def _requested_router(kwargs: Mapping[str, object]) -> "Router | None":
     from litellm.router import Router
+    from litellm.router_python_core import PythonCore
 
     requested: Final = kwargs.get("litellm_router")
-    return requested if isinstance(requested, Router) else None
+    return requested if isinstance(requested, (Router, PythonCore)) else None  # pyright: ignore[reportReturnType]  # the router passes its core, which carries Router's members
 
 
 def request_safeguards_evaluator(

litellm/llms/bedrock/chat/agentcore/transformation.py

@@ -544,6 +544,17 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
 
         return AgentCoreParsedResponse(content=content, usage=usage_data, final_message=final_message)
 
+    @staticmethod
+    def _message_stop_chunk(model: str) -> ModelResponseStream:
+        chunk: Final = ModelResponseStream(
+            id=f"chatcmpl-{uuid.uuid4()}",
+            created=0,
+            model=model,
+            object="chat.completion.chunk",
+        )
+        chunk.choices = [StreamingChoices(finish_reason="stop", index=0, delta=Delta())]
+        return chunk
+
     def _stream_agentcore_response_sync(
         self,
         response: httpx.Response,
@@ -553,6 +564,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         Internal sync generator that parses SSE and yields ModelResponse chunks.
         """
         buffer = ""
+        sent_finish = False  # rebind-ok: flips when a finish chunk goes out
+        message_stopped = False  # rebind-ok: flips on messageStop, the only end some runtimes send
         for text_chunk in response.iter_text():
             buffer += text_chunk
 
@@ -624,7 +637,9 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
                                     total_tokens=usage_data.get("totalTokens", 0),
                                 ),
                             )
+                            sent_finish = True
                             yield chunk
+                        message_stopped |= "messageStop" in event_payload
 
                     # Process final message
                     if "message" in data_obj and isinstance(data_obj["message"], dict):
@@ -641,12 +656,16 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
                                 delta=Delta(),
                             )
                         ]
+                        sent_finish = True
                         yield chunk
 
                 except json.JSONDecodeError:
                     verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100])
                     continue
 
+        if message_stopped and not sent_finish:
+            yield self._message_stop_chunk(model)
+
     def get_sync_custom_stream_wrapper(
         self,
         model: str,
@@ -775,6 +794,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         Internal async generator that parses SSE and yields ModelResponse chunks.
         """
         buffer = ""
+        sent_finish = False  # rebind-ok: flips when a finish chunk goes out
+        message_stopped = False  # rebind-ok: flips on messageStop, the only end some runtimes send
         async for text_chunk in response.aiter_text():
             buffer += text_chunk
 
@@ -846,7 +867,9 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
                                     total_tokens=usage_data.get("totalTokens", 0),
                                 ),
                             )
+                            sent_finish = True
                             yield chunk
+                        message_stopped |= "messageStop" in event_payload
 
                     # Process final message
                     if "message" in data_obj and isinstance(data_obj["message"], dict):
@@ -863,12 +886,16 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
                                 delta=Delta(),
                             )
                         ]
+                        sent_finish = True
                         yield chunk
 
                 except json.JSONDecodeError:
                     verbose_logger.debug("Skipping non-JSON SSE line: %s", line[:100])
                     continue
 
+        if message_stopped and not sent_finish:
+            yield self._message_stop_chunk(model)
+
     async def get_async_custom_stream_wrapper(
         self,
         model: str,

litellm/llms/deepgram/common_utils.py

@@ -1,16 +1,21 @@
+import json
 import math
+import re
 from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
 from types import MappingProxyType
 from typing import Final
-from urllib.parse import parse_qs, urlparse
+from urllib.parse import parse_qs, urlparse, urlsplit, urlunsplit
 
 import httpx
 
 import litellm
-from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL
+from litellm.constants import DEEPGRAM_DEFAULT_API_BASE, DEEPGRAM_LISTEN_DEFAULT_MODEL, DEEPGRAM_SPEAK_DEFAULT_MODEL
 from litellm.litellm_core_utils.url_utils import join_api_base
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.types.passthrough_endpoints.pass_through_endpoints import WebSocketSessionUsage
 from litellm.types.utils import LlmProviders
+from litellm.utils import ModelInfoLookupMiss, lookup_model_info_for_registration
 
 _WEBSOCKET_SCHEMES: Final = MappingProxyType({"https": "wss", "http": "ws", "wss": "wss", "ws": "ws"})
 DEEPGRAM_LISTEN_CALLBACK_PARAMS: Final = frozenset({"callback", "callback_method"})
@@ -199,3 +204,71 @@ def deepgram_listen_transcript(websocket_messages: Sequence[Mapping[str, object]
         if frame.get("type") == "Results"
         if (transcript := _final_transcript(frame)) is not None
     )
+
+
+_AURA_GENERATION: Final = re.compile(r"aura(?:-\d+)?(?=-[a-z])")
+
+
+def deepgram_speak_websocket_target(api_base: str | None, query_string: str) -> str:
+    speak_url: Final = urlsplit(join_api_base(api_base or DEEPGRAM_DEFAULT_API_BASE, "speak"))
+    scheme: Final = _WEBSOCKET_SCHEMES.get(speak_url.scheme, speak_url.scheme)
+    params: Final = _first_occurrences(query_string)
+    query: Final = params if params.get("model") else params.remove("model").add("model", DEEPGRAM_SPEAK_DEFAULT_MODEL)
+    return urlunsplit((scheme, speak_url.netloc, speak_url.path, str(query), ""))
+
+
+def deepgram_speak_model(upstream_url: str) -> str:
+    models: Final = parse_qs(urlparse(upstream_url).query).get("model")
+    return models[0] if models else DEEPGRAM_SPEAK_DEFAULT_MODEL
+
+
+def deepgram_speak_requested_model(query_string: str) -> str:
+    """Read off the upstream target, so the voice a key is authorized for is the voice Deepgram receives"""
+    return deepgram_speak_model(deepgram_speak_websocket_target(None, query_string))
+
+
+def deepgram_speak_registry_key(upstream_url: str) -> str:
+    """Deepgram bills every voice of an Aura generation at one rate, so ``aura-2-thalia-en`` is priced at the
+    ``deepgram/aura-2`` row"""
+    model: Final = deepgram_speak_model(upstream_url)
+    generation: Final = _AURA_GENERATION.match(model)
+    return f"{LlmProviders.DEEPGRAM.value}/{generation.group() if generation else model}"
+
+
+def deepgram_speak_character_rate(upstream_url: str) -> float | None:
+    """Only an exact registry hit counts, so a voice is never billed at a neighbouring row's rate"""
+    registry_key: Final = deepgram_speak_registry_key(upstream_url)
+    model_info: Final = lookup_model_info_for_registration(registry_key, LlmProviders.DEEPGRAM.value)
+    if isinstance(model_info, ModelInfoLookupMiss) or model_info["key"] != registry_key:
+        return None
+    return model_info.get("input_cost_per_character")
+
+
+@dataclass(frozen=True, slots=True)
+class DeepgramSpeakMeter:
+    """Holds the rate checked when the socket opened, so a cost-map reload mid-session can't change or drop the price"""
+
+    upstream_url: str
+    character_rate: float
+
+    def measure(self, client_text: str) -> int:
+        """Counts a Speak message's text as it is sent. Deepgram also speaks, and bills, unflushed text on its own (a
+        long buffer, Close, a close frame), so counting only flushed text would let a client get audio free, while
+        counting on send can only over-bill text Deepgram discards, such as a Clear or an oversized Speak. Never
+        raises, so a malformed frame can't end the session"""
+        try:
+            message: Final[object] = json.loads(client_text)
+        except (ValueError, RecursionError):
+            return 0
+        match message:
+            case {"type": "Speak", "text": str() as text}:
+                return len(text)
+            case _:
+                return 0
+
+    def price(self, total: int) -> WebSocketSessionUsage:
+        return WebSocketSessionUsage(
+            model=deepgram_speak_model(self.upstream_url),
+            custom_llm_provider=LlmProviders.DEEPGRAM.value,
+            response_cost=total * self.character_rate,
+        )

litellm/llms/langgraph/chat/sse_iterator.py

@@ -41,8 +41,9 @@ class LangGraphSSEStreamIterator:
         return self
 
     def __aiter__(self):
-        """Initialize async iteration."""
-        self.async_line_iterator = aiter_sse_lines(self.response.aiter_text())
+        """Initialize async iteration once; CustomStreamWrapper re-enters `async for` on every chunk."""
+        if self.async_line_iterator is None:
+            self.async_line_iterator = aiter_sse_lines(self.response.aiter_text())
         return self
 
     def _parse_sse_line(self, line: str) -> ModelResponseStream | None:

litellm/llms/oci/chat/transformation.py

@@ -20,7 +20,11 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
 from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
-from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
+from litellm.litellm_core_utils.prompt_templates.common_utils import (
+    merge_consecutive_system_messages,
+    messages_with_file_parts_as_image_urls,
+    system_messages_first,
+)
 from litellm.litellm_core_utils.reasoning_effort_capability import declared_reasoning_efforts_for_model
 from litellm.litellm_core_utils.reasoning_effort_utils import warn_dropped_reasoning
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@@ -693,14 +697,18 @@ class OCIChatConfig(BaseConfig):
                 chatRequest=chat_request,
             )
         else:
+            generic_messages: Final = list(messages_with_file_parts_as_image_urls(messages))
+            normalized_messages: Final = (
+                merge_consecutive_system_messages(system_messages_first(generic_messages))
+                if model.lower().removeprefix("oci/").startswith("google.gemini")
+                else generic_messages
+            )
             data = OCICompletionPayload(
                 compartmentId=oci_compartment_id,
                 servingMode=serving_mode,
                 chatRequest=OCIChatRequestPayload(
                     apiFormat=vendor.value,
-                    messages=adapt_messages_to_generic_oci_standard(
-                        list(messages_with_file_parts_as_image_urls(messages))
-                    ),
+                    messages=adapt_messages_to_generic_oci_standard(normalized_messages),
                     **self._get_optional_params(vendor, optional_params, model),
                 ),
             )

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -40,7 +40,11 @@ from typing import TYPE_CHECKING, Any, Final, NamedTuple, Protocol, TypeVar, Uni
 
 from openai.types.responses.response_output_refusal import ResponseOutputRefusal
 from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
-from typing_extensions import ReadOnly, TypedDict
+from typing_extensions import (
+    ReadOnly,
+    TypedDict,
+    TypeIs,  # noqa: TID251  # TypeIs reaches typing only on 3.13
+)
 
 from litellm._logging import verbose_proxy_logger
 from litellm.completion_extras.litellm_responses_transformation.transformation import (
@@ -100,6 +104,7 @@ from litellm.types.llms.openai import (
     OutputTextDeltaEvent,
     OutputTextDoneEvent,
     ResponseAPIUsage,
+    ResponseCompactionItem,
     ResponseCompletedEvent,
     ResponsesAPIOptionalRequestParams,
     ResponsesAPIResponse,
@@ -712,6 +717,15 @@ def _next_stream_sequence_number(responses_so_far: Sequence[object] | None) -> i
     return next((n + 1 for n in sequence_numbers if isinstance(n, int)), 0)
 
 
+def _is_responses_input(
+    input_data: object,
+) -> TypeIs["str | ResponseInputParam"]:  # guard-ok: str, or a list of Mapping items
+    if isinstance(input_data, str):
+        return True
+    items: Final = cast(list[object], input_data)  # cast-ok: isinstance narrows to list[Unknown] only
+    return isinstance(input_data, list) and all(isinstance(item, (Mapping, ResponseCompactionItem)) for item in items)
+
+
 class OpenAIResponsesHandler(BaseTranslation):
     """
     Handler for processing OpenAI Responses API with guardrails.
@@ -733,8 +747,8 @@ class OpenAIResponsesHandler(BaseTranslation):
         Transforms `input` (string or ResponseInputParam) and optional
         `instructions` into chat completion messages.
         """
-        input_data: Final = data.get("input")
-        if input_data is None:
+        input_data: Final[object] = data.get("input")
+        if not _is_responses_input(input_data):
             return None
         messages: Final = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
             input=input_data,

litellm/llms/vertex_ai/batches/transformation.py

@@ -129,6 +129,17 @@ def _native_vertex_row_usage(
     return calculate_usage(completion_response)
 
 
+def native_vertex_batch_row_model(row: Mapping[str, object], model_name: str | None) -> str | None:
+    response: Final = as_json_object(row.get("response"))
+    model_version: Final = (
+        response.get("model" if is_vertex_anthropic_batch_output_response(response) else "modelVersion")
+        if response is not None
+        else None
+    )
+    deployment_model: Final = model_name if model_name and "*" not in model_name else None
+    return deployment_model or (model_version if isinstance(model_version, str) else model_name)
+
+
 def native_vertex_batch_row_stats(
     row: Mapping[str, object],
     model_name: str | None,
@@ -156,11 +167,7 @@ def native_vertex_batch_row_stats(
     if usage is None:
         return None
     total_tokens: Final = usage.total_tokens or (usage.prompt_tokens + usage.completion_tokens)
-    model_version: Final = response_body.get(
-        "model" if is_vertex_anthropic_batch_output_response(response_body) else "modelVersion"
-    )
-    deployment_model: Final = model_name if model_name and "*" not in model_name else None
-    model: Final = deployment_model or (model_version if isinstance(model_version, str) else model_name)
+    model: Final = native_vertex_batch_row_model(row, model_name)
     if model is None:
         verbose_logger.warning(
             "vertex_ai batch output row could not be costed, so it is billed at $0 and the rest of the batch "

litellm/main.py

@@ -1237,6 +1237,7 @@ def mock_completion(
                     custom_llm_provider="openai",
                     logging_obj=logging,
                     count_prompt_tokens=count_prompt_tokens,
+                    is_mock=True,
                 )
             return CustomStreamWrapper(
                 completion_stream=mock_completion_streaming_obj(
@@ -1246,6 +1247,7 @@ def mock_completion(
                 custom_llm_provider="openai",
                 logging_obj=logging,
                 count_prompt_tokens=count_prompt_tokens,
+                is_mock=True,
             )
         if isinstance(mock_response, litellm.MockException):
             raise mock_response

litellm/model_prices_and_context_window_backup.json

@@ -81372,5 +81372,77 @@
         "supports_response_schema": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true
+    },
+    "gpt-6-astra-law": {
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "openai",
+        "max_input_tokens": 922000,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "reasoning_contexts": [
+            "auto",
+            "current_turn",
+            "all_turns"
+        ],
+        "reasoning_modes": [
+            "standard",
+            "pro"
+        ],
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
+        "source": "https://developers.openai.com/api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": false,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_cache_breakpoint": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_tool_search": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "chatgpt/gpt-6-astra-law": {
+        "litellm_provider": "chatgpt",
+        "max_input_tokens": 922000,
+        "max_output_tokens": 128000,
+        "mode": "responses",
+        "source": "https://developers.openai.com/api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_none_reasoning_effort": false,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
     }
 }

litellm/proxy/_types.py

@@ -38,12 +38,14 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
     validate_no_callback_env_reference,
 )
 from litellm.proxy.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled
+from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
 from litellm.types.agents import (
     AgentCaller as AgentCaller,  # noqa: PLC0414  # public re-export
 )
 from litellm.types.agents import (
     AgentResponse as AgentResponse,  # noqa: PLC0414  # public re-export
 )
+from litellm.types.decisions import OpenAIDecisionResponse
 from litellm.types.integrations.compression_interception import (
     CompressionSavingsMetadata,
 )
@@ -2470,6 +2472,22 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
     Documents all the fields supported by `general_settings` in config.yaml
     """
 
+    accounting_capture: UsageCaptureSettings | None = Field(
+        None,
+        description="Opt-in durable usage capture. Legacy spend updates remain active. Requires local Prisma writes.",
+    )
+    accounting_sync_ack: bool = Field(
+        False,
+        description="Wait for usage commit on foreground non-stream chat, text, embedding and Responses requests. "
+        "Other requests remain supported with asynchronous usage capture.",
+    )
+    accounting_sync_ack_timeout_seconds: float = Field(
+        20.0, gt=0, description="Maximum response wait for usage commit, including the batch flush interval"
+    )
+    accounting_sync_ack_failure_policy: Literal["fail_closed", "log"] = Field(
+        "fail_closed", description="Return 503 or log the persistence failure after the response wait expires"
+    )
+
     request_tag_mappings: Sequence[RequestTagMapping] | None = Field(
         None,
         description="Map W3C baggage values to request tags. Exact value/tag rules take precedence over tag_prefix fallbacks.",
@@ -4584,6 +4602,7 @@ PassThroughEndpointLoggingResultValues = (
     | StandardPassThroughResponseObject
     | ResponsesAPIResponse
     | TranscriptionResponse
+    | OpenAIDecisionResponse
 )
 
 

litellm/proxy/agent_endpoints/a2a_endpoints.py

@@ -11,7 +11,7 @@ The A2A SDK can point to LiteLLM's URL and invoke agents registered with LiteLLM
 """
 
 import json
-from collections.abc import AsyncGenerator, Mapping
+from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping
 from copy import deepcopy
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Protocol
@@ -19,7 +19,8 @@ from urllib.parse import urlparse
 
 from fastapi import APIRouter, Depends, HTTPException, Request, Response
 from fastapi.responses import JSONResponse, StreamingResponse
-from pydantic import TypeAdapter, ValidationError
+from fastapi.routing import APIRoute
+from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -27,7 +28,7 @@ from litellm.litellm_core_utils.core_helpers import as_str_mapping
 from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
 from litellm.llms.a2a.common_utils import resolve_a2a_hop_auth_header
 from litellm.llms.custom_httpx.sse_lines import aiter_sse_lines
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy._types import ProxyException, UserAPIKeyAuth
 from litellm.proxy.a2a.version_convert import (
     A2AVersion,
     normalize_agent_card,
@@ -42,6 +43,7 @@ from litellm.proxy.agent_endpoints.databricks_oauth import (
 )
 from litellm.proxy.agent_endpoints.utils import merge_agent_headers
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.common_utils.openai_error_payload import status_typed_proxy_exception
 from litellm.proxy.common_utils.sse_keepalive import (
     SSE_COMMENT_PING,
     coerce_keepalive_interval,
@@ -172,13 +174,16 @@ def _caller_identity_headers(user_api_key_dict: UserAPIKeyAuth) -> Mapping[str,
 async def _resolve_backend_auth(
     litellm_params: dict[str, object],
     custom_llm_provider: object,
+    agent_extra_headers: Mapping[str, str] | None = None,
 ) -> AgentBackendAuth:
     databricks_auth: Final = await resolve_databricks_backend_auth(litellm_params)
     if has_databricks_oauth(litellm_params):
         return databricks_auth
     return AgentBackendAuth(
         litellm_params=databricks_auth.litellm_params,
-        header=await resolve_a2a_hop_auth_header(litellm_params, custom_llm_provider),
+        header=await resolve_a2a_hop_auth_header(
+            litellm_params, custom_llm_provider, agent_extra_headers=agent_extra_headers
+        ),
     )
 
 
@@ -218,6 +223,57 @@ def _jsonrpc_error(
     )
 
 
+class _JsonRpcRequestId(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    id: str | int | None = None
+
+
+async def _jsonrpc_request_id(request: Request) -> str | int | None:
+    try:
+        return _JsonRpcRequestId.model_validate_json(await request.body()).id
+    except ValidationError:
+        return None
+
+
+async def _jsonrpc_refusal(request: Request, refusal: ProxyException | HTTPException) -> JSONResponse:
+    from litellm.proxy.proxy_server import (
+        _close_dangling_otel_server_span,  # pyright: ignore[reportPrivateUsage]  # proxy_server keeps the span-close helper private; a route that answers an error itself must stamp the OTel server span like the global ProxyException handler does
+    )
+
+    error: Final = refusal if isinstance(refusal, ProxyException) else status_typed_proxy_exception(refusal)
+    _close_dangling_otel_server_span(request, int(error.code), exc=refusal)
+    return JSONResponse(
+        content={
+            "jsonrpc": "2.0",
+            "id": await _jsonrpc_request_id(request),
+            "error": {
+                "code": -32000,
+                "message": error.message,
+                "data": {"type": error.type, "param": error.param, "code": error.code},
+            },
+        },
+        status_code=int(error.code),
+        headers=error.headers if isinstance(refusal, ProxyException) else refusal.headers,
+    )
+
+
+class _JsonRpcRefusalRoute(APIRoute):
+    def get_route_handler(self) -> Callable[[Request], Coroutine[object, object, Response]]:
+        handler: Final = super().get_route_handler()
+
+        async def answer_refusals_as_jsonrpc(request: Request) -> Response:
+            try:
+                return await handler(request)
+            except (ProxyException, HTTPException) as refusal:
+                return await _jsonrpc_refusal(request, refusal)
+
+        return answer_refusals_as_jsonrpc
+
+
+jsonrpc_router: Final = APIRouter(route_class=_JsonRpcRefusalRoute)
+
+
 async def _get_agent(agent_id: str) -> "AgentResponse | None":
     """Look up an agent by ID or name. Returns None if not found."""
     from litellm.proxy.common_utils.registry_read_through import (
@@ -646,17 +702,17 @@ async def get_agent_card(
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@router.post(
+@jsonrpc_router.post(
     "/a2a/{agent_id}",
     tags=["[beta] A2A Agents"],
     dependencies=[Depends(user_api_key_auth)],
 )
-@router.post(
+@jsonrpc_router.post(
     "/a2a/{agent_id}/message/send",
     tags=["[beta] A2A Agents"],
     dependencies=[Depends(user_api_key_auth)],
 )
-@router.post(
+@jsonrpc_router.post(
     "/v1/a2a/{agent_id}/message/send",
     tags=["[beta] A2A Agents"],
     dependencies=[Depends(user_api_key_auth)],
@@ -834,14 +890,15 @@ async def invoke_agent_a2a(
                     if header_name:
                         dynamic_headers[header_name] = val
 
-        backend_auth: Final = await _resolve_backend_auth(litellm_params, custom_llm_provider)
+        merged_agent_headers: Final = merge_agent_headers(
+            dynamic_headers=dynamic_headers or None,
+            static_headers=static_headers or None,
+        )
+        backend_auth: Final = await _resolve_backend_auth(litellm_params, custom_llm_provider, merged_agent_headers)
         agent_extra_headers: Final = _forwarding_headers(
             caller_identity=caller_identity,
             request_data=data,
-            agent_extra_headers=merge_agent_headers(
-                dynamic_headers=dynamic_headers or None,
-                static_headers=static_headers or None,
-            ),
+            agent_extra_headers=merged_agent_headers,
             backend_auth_header=backend_auth.header,
         )
         forwarded_litellm_params: Final = backend_auth.litellm_params
@@ -1051,3 +1108,6 @@ async def invoke_agent_a2a(
         if isinstance(e, litellm.BadRequestError):
             return _jsonrpc_error(body.get("id"), -32602, e.message, 400)
         return _jsonrpc_error(body.get("id"), -32603, f"Internal error: {e}", 500)
+
+
+router.include_router(jsonrpc_router)

litellm/proxy/auth/auth_checks.py

@@ -1086,13 +1086,31 @@ def model_max_budget_of(holder: ModelMaxBudgetHolder) -> Mapping[str, object] |
 
 
 def route_skips_budget_checks(route: str) -> bool:
-    return route not in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES and (
-        route in MODEL_DISCOVERY_ROUTES or not RouteChecks.is_llm_api_route(route=route)
+    if route in BUDGET_ENFORCED_SIDE_EFFECT_ROUTES:
+        return False
+    if route in MODEL_DISCOVERY_ROUTES:
+        return True
+    if RouteChecks.is_llm_api_route(route=route):
+        return False
+    from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (  # noqa: PLC0415  # pass-through endpoints import the auth stack
+        InitPassThroughEndpointHelpers,
     )
 
+    return not InitPassThroughEndpointHelpers.is_registered_pass_through_route(route=route)
 
-def request_skips_budget_checks(route: str, model: str | list[str] | None, llm_router: Router | None) -> bool:
-    return route_skips_budget_checks(route=route) or is_model_cost_zero(model=model, llm_router=llm_router)
+
+def request_skips_budget_checks(
+    route: str,
+    model: str | list[str] | None,
+    llm_router: Router | None,
+    *,
+    request_body: Mapping[str, object] | None = None,
+) -> bool:
+    if route_skips_budget_checks(route=route):
+        return True
+    if request_body is not None and is_mcp_discovery_request(route=route, request_body=request_body):
+        return True
+    return is_model_cost_zero(model=model, llm_router=llm_router)
 
 
 async def common_checks(
@@ -1142,10 +1160,11 @@ async def common_checks(
         team_id=valid_token.team_id if valid_token is not None else None,
     )
 
-    skip_all_budget_checks: Final = (
-        skip_budget_checks
-        or route_skips_budget_checks(route=route)
-        or is_mcp_discovery_request(route=route, request_body=_typed_request_body(request_body))
+    skip_all_budget_checks: Final = skip_budget_checks or request_skips_budget_checks(
+        route=route,
+        model=_model,
+        llm_router=llm_router,
+        request_body=_typed_request_body(request_body),
     )
 
     membership_user_id: Final = (

litellm/proxy/auth/user_api_key_auth.py

@@ -80,7 +80,6 @@ from litellm.proxy.auth.auth_checks import (  # noqa: F401  # legacy module expo
     get_team_object,
     get_user_object,
     get_user_role,
-    is_mcp_discovery_request,
     is_model_cost_zero,
     is_user_proxy_admin,
     is_valid_fallback_model,
@@ -88,6 +87,7 @@ from litellm.proxy.auth.auth_checks import (  # noqa: F401  # legacy module expo
     key_access_group_grants_model,
     key_model_aliases_for_auth_check,
     model_max_budget_of,
+    request_skips_budget_checks,
     resolve_and_validate_end_user_id,
     resolve_default_end_user_budget,
     route_skips_budget_checks,
@@ -2497,15 +2497,9 @@ async def user_api_key_auth_builder(
                             llm_router=llm_router,
                             team_id=valid_token.team_id,
                         )
-                        skip_budget_checks = is_mcp_discovery_request(route=route, request_body=request_data)
-                        if not skip_budget_checks and model is not None and llm_router is not None:
-                            from litellm.proxy.auth.auth_checks import is_model_cost_zero
-
-                            skip_budget_checks = (  # rebind-ok: pre-existing rebinding on a rename-only line
-                                is_model_cost_zero(model=model, llm_router=llm_router)
-                            )
-                            if skip_budget_checks:
-                                verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model)
+                        skip_budget_checks = request_skips_budget_checks(
+                            route=route, model=model, llm_router=llm_router, request_body=request_data
+                        )
 
                         # Fetch project object for JWT path if project_id is set
                         _jwt_project_obj = None
@@ -3019,15 +3013,9 @@ async def validate_resolved_virtual_key(  # noqa: C901  # Preserve ordering of e
             llm_router=llm_router,
             team_id=valid_token.team_id,
         )
-        skip_budget_checks = is_mcp_discovery_request(route=route, request_body=request_data)
-        if not skip_budget_checks and model is not None and llm_router is not None:
-            from litellm.proxy.auth.auth_checks import is_model_cost_zero
-
-            skip_budget_checks = is_model_cost_zero(  # rebind-ok: pre-existing rebinding on a rename-only line
-                model=model, llm_router=llm_router
-            )
-            if skip_budget_checks:
-                verbose_proxy_logger.info("Skipping all budget checks for zero-cost model: %s", model)
+        skip_budget_checks: Final = request_skips_budget_checks(
+            route=route, model=model, llm_router=llm_router, request_body=request_data
+        )
 
         # Check 3. Check if user is in their team budget
         if not skip_budget_checks and valid_token.team_member_spend is not None:
@@ -4088,11 +4076,7 @@ def _should_skip_budget_checks(
         llm_router=llm_router,
         team_id=team_id,
     )
-    if is_mcp_discovery_request(route=route, request_body=request_data):
-        return True
-    if model is not None and llm_router is not None:
-        return is_model_cost_zero(model=model, llm_router=llm_router)
-    return False
+    return request_skips_budget_checks(route=route, model=model, llm_router=llm_router, request_body=request_data)
 
 
 def _resolve_request_principal(request: Request, valid_token: UserAPIKeyAuth) -> Principal:

litellm/proxy/common_request_processing.py

@@ -137,6 +137,7 @@ from litellm.proxy.route_llm_request import (
     route_request,
 )
 from litellm.proxy.spend_tracking.cost_finalization import finalize_proxy_response_cost, prepare_response_cost
+from litellm.proxy.spend_tracking.usage_capture import capture_proxy_response
 from litellm.proxy.telemetry.request_context import bind_call_id as bind_telemetry_call_id
 from litellm.proxy.utils import (  # noqa: F401  # legacy module exports
     ProxyLogging,
@@ -3366,6 +3367,9 @@ class ProxyBaseLLMRequestProcessing:
                     )
                 ),
             )
+            await capture_proxy_response(
+                logging_obj, response, background=_REQUEST_FIELDS.validate_python(self.data).get("background") is True
+            )
         except (Exception, asyncio.CancelledError) as exc:
             _exception_raised = True
             if logging_obj.prepared_cost is not None:

litellm/proxy/db/accounting_ledger.py

@@ -275,7 +275,7 @@ class AccountingLedger:
             )
         return tuple(
             UsageWrite(
-                OutboundUsageEvent.model_validate(_JSON.validate_python(row["payload"])["event"]),
+                OutboundUsageEvent.model_validate_json(_json(_JSON.validate_python(row["payload"])["event"])),
                 _json(row["payload"]),
                 TypeAdapter(tuple[DeliveryTarget, ...]).validate_python(row["targets"]),
             )

litellm/proxy/hooks/proxy_track_cost_callback.py

@@ -22,7 +22,11 @@ from litellm.litellm_core_utils.core_helpers import (
     get_metadata_variable_name_from_kwargs,
     get_parent_otel_span_from_kwargs,
 )
-from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+from litellm.litellm_core_utils.litellm_logging import (
+    StandardLoggingPayloadSetup,
+    UsageCaptureObservation,
+    UsageObservation,
+)
 from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import guardrail_information_cost
 from litellm.litellm_core_utils.redact_messages import MESSAGE_REDACTION_ENABLE_HEADERS
 from litellm.llms.azure.ptu_capacity import is_str_keyed_mapping
@@ -63,6 +67,7 @@ from litellm.proxy.spend_tracking.spend_tracking_utils import (  # noqa: F401  #
     sanitize_error_information_for_spend_logs,
     should_store_prompts_and_responses_in_spend_logs,
 )
+from litellm.proxy.spend_tracking.usage_capture import CapturedUsage, capture_callback, captured_callback_kwargs
 from litellm.proxy.utils import (
     REJECTED_BEFORE_REQUEST_SETUP,
     REJECTED_BEFORE_REQUEST_SETUP_KEY,
@@ -344,11 +349,27 @@ def _attempt_spend_row_kwargs(
         "litellm_params": litellm_params,
         "combined_usage_object": attempt.usage,
         "response_cost": attempt.response_cost,
+        "_accounting_usage_observed": UsageObservation.ESTIMATED,
+        "_accounting_usage_snapshot": UsageCaptureObservation.from_usage(
+            attempt.usage, UsageObservation.ESTIMATED, attempt.response_cost
+        ),
+        "_accounting_prior_captures": (),
     }
     row.pop("billed_failed_attempts", None)
+    row.pop("_accounting_capture", None)
     call_id: Final = request_data.get("litellm_call_id") or litellm_params.get("litellm_call_id")
     if isinstance(call_id, str):
         row["litellm_call_id"] = f"{call_id}{suffix}"
+        row["parent_request_id"] = call_id
+        metadata["parent_request_id"] = call_id
+        captured_attempts: Final = request_data.get("_accounting_billed_attempts")
+        if isinstance(captured_attempts, tuple):
+            for captured in TypeAdapter(tuple[object, ...]).validate_python(captured_attempts):
+                if (
+                    isinstance(captured, CapturedUsage)
+                    and captured.payload["litellm_call_id"] == row["litellm_call_id"]
+                ):
+                    row["_accounting_capture"] = captured
     standard_logging_object: Final = request_data.get("standard_logging_object")
     if isinstance(standard_logging_object, dict):
         payload: Final = cast("Mapping[str, object]", standard_logging_object)  # cast-ok: StandardLoggingPayload dict
@@ -385,7 +406,7 @@ class _ProxyDBLogger(CustomLogger):
         self, kwargs: ObjectMapping, response_obj: object, start_time: datetime, end_time: datetime
     ) -> None:
         if self.spend_event_producer is None or not is_offloadable_success(response_obj):
-            await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time)
+            await self._PROXY_track_cost_callback(dict(kwargs), response_obj, start_time, end_time)
             return
         event: Final = build_spend_event(
             kwargs,
@@ -396,7 +417,7 @@ class _ProxyDBLogger(CustomLogger):
         )
         if isinstance(event, SpendEventBuildError):
             verbose_proxy_logger.warning("collector: tracking cost in-process, event not buildable: %s", event.reason)
-            await self._PROXY_track_cost_callback(kwargs, response_obj, start_time, end_time)
+            await self._PROXY_track_cost_callback(dict(kwargs), response_obj, start_time, end_time)
             return
         await self.spend_event_producer.publish(event)
 
@@ -434,6 +455,9 @@ class _ProxyDBLogger(CustomLogger):
                     "Failed to invalidate budget reservation counters after failure release failed"
                 )
 
+        capture_metadata: Final = await self._try_capture_failure_usage(
+            request_data, original_exception, user_api_key_dict
+        )
         if completed_cost and tracking_disabled:
             return
 
@@ -477,8 +501,12 @@ class _ProxyDBLogger(CustomLogger):
         )
         _metadata["error_information"] = _error_information
 
-        _metadata = await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled(  # rebind-ok: pre-existing rebinding on a rename-only line
-            metadata=_metadata, original_exception=original_exception
+        _metadata = (  # rebind-ok: enriched identity is shared with the earlier immutable capture
+            {**_metadata, **capture_metadata}
+            if capture_metadata is not None
+            else await ProxyDBLogger._enrich_failure_metadata_unless_db_stalled(
+                metadata=_metadata, original_exception=original_exception
+            )
         )
 
         request_fields: Final = _str_keyed_fields(request_data)
@@ -595,6 +623,106 @@ class _ProxyDBLogger(CustomLogger):
             ),
         )
 
+    async def _failure_capture_metadata(
+        self, request_data: Mapping[str, object], exception: Exception, auth: UserAPIKeyAuth
+    ) -> Mapping[str, object] | None:
+        from litellm.proxy.proxy_server import usage_capture
+
+        if usage_capture.meter is None:
+            return None
+        captured_identity: Final = (
+            _batch_accounting_for(request_data.get("litellm_call_id")) is not None
+            and request_data.get(REJECTED_BEFORE_REQUEST_SETUP_KEY) is not REJECTED_BEFORE_REQUEST_SETUP
+        )
+        current_identity: Final = {
+            **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=auth),
+            "user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(auth),
+        }
+        metadata: Final = (
+            {
+                **dict.fromkeys(current_identity),
+                **{
+                    key: value
+                    for key, value in _REQUEST_DATA.validate_python(
+                        get_litellm_metadata_from_kwargs(dict(request_data))
+                    ).items()
+                    if key in current_identity or key == "tags"
+                },
+            }
+            if captured_identity
+            else current_identity
+        )
+        return await self._enrich_failure_metadata_unless_db_stalled(
+            metadata=metadata, original_exception=exception, resolve_missing_key_identity=not captured_identity
+        )
+
+    async def _try_capture_failure_usage(
+        self,
+        request_data: dict[str, object],  # mutable-ok: receipt handoff shared with the legacy writer
+        original_exception: Exception,
+        auth: UserAPIKeyAuth,
+    ) -> Mapping[str, object] | None:
+        try:
+            metadata: Final = await self._failure_capture_metadata(request_data, original_exception, auth)
+            self._capture_failure_usage(request_data, original_exception, auth, metadata)
+        except Exception:  # noqa: BLE001  # capture failure must not suppress legacy billing or reservation release
+            verbose_proxy_logger.exception("Usage capture failed on the request failure path")
+            return None
+        return metadata
+
+    def _capture_failure_usage(
+        self,
+        request_data: dict[str, object],  # mutable-ok: receipt handoff shared with the legacy writer
+        original_exception: Exception,
+        auth: UserAPIKeyAuth,
+        metadata: Mapping[str, object] | None = None,
+    ) -> None:
+        from litellm.litellm_core_utils.litellm_logging import Logging
+        from litellm.proxy.db.exception_handler import exception_chain
+        from litellm.proxy.proxy_server import usage_capture
+        from litellm.proxy.spend_tracking.usage_capture import UsageAcknowledgementError
+
+        if any(isinstance(error, UsageAcknowledgementError) for error in exception_chain(original_exception)):
+            return
+        usage_meter: Final = usage_capture.meter
+
+        if usage_meter is None:
+            return
+        logging_obj: Final = request_data.get("litellm_logging_obj")
+        details: Final[Mapping[str, object]] = (
+            _str_keyed_fields(logging_obj.model_call_details) if isinstance(logging_obj, Logging) else {}
+        )
+        params: Final = _str_keyed_fields(request_data.get("litellm_params") or details.get("litellm_params"))
+        failure_data: Final = {
+            **details,
+            **request_data,
+            "litellm_params": {
+                **params,
+                "metadata": {
+                    **_str_keyed_fields(params.get("metadata")),
+                    **(
+                        metadata
+                        if metadata is not None
+                        else {
+                            **LiteLLMProxyRequestSetup.get_sanitized_user_information_from_key(user_api_key_dict=auth),
+                            "user_api_key": LiteLLMProxyRequestSetup.get_logged_api_key(auth),
+                        }
+                    ),
+                    "status": "failure",
+                },
+            },
+        }
+        start: Final = logging_obj.start_time if isinstance(logging_obj, Logging) else request_data.get("start_time")
+        if not isinstance(start, datetime):
+            return
+        end: Final = datetime.now(timezone.utc)
+        captured: Final = tuple(
+            capture_callback(usage_meter, row, original_exception, start, end)
+            for _, row in _failure_spend_rows(failure_data, 0.0, None, self._deployment_lookup)
+        )
+        request_data["_accounting_capture"] = captured[0]  # rebind-ok: share one receipt with the legacy writer
+        request_data["_accounting_billed_attempts"] = captured[1:]  # rebind-ok: share attempt receipts
+
     async def _write_failure_spend_rows(
         self,
         user_api_key_dict: UserAPIKeyAuth,
@@ -780,6 +908,7 @@ class _ProxyDBLogger(CustomLogger):
                     resolve_missing_key_identity=str(kwargs.get("call_type")) not in _CAPTURED_IDENTITY_CALL_TYPES,
                 )
                 _write_spend_metadata_to_kwargs(kwargs=kwargs, metadata=metadata)
+            kwargs.update(await captured_callback_kwargs(kwargs, completion_response, start_time, end_time))
             budget_reservation: Final = _get_budget_reservation_from_metadata(metadata=metadata)
             if accounting is not None and (budget_reservation is not None or litellm.apply_search_free_tier):
                 accounting.write_started = True
@@ -967,11 +1096,13 @@ class _ProxyDBLogger(CustomLogger):
 
     @staticmethod
     async def _enrich_failure_metadata_unless_db_stalled(
-        metadata: dict[str, object], original_exception: Exception
+        metadata: dict[str, object], original_exception: Exception, resolve_missing_key_identity: bool = True
     ) -> dict[str, object]:
         if isinstance(original_exception, DBLookupDeadlineExceeded):
             return metadata
-        return await ProxyDBLogger._enrich_failure_metadata_with_key_info(metadata=metadata)
+        return await ProxyDBLogger._enrich_failure_metadata_with_key_info(
+            metadata=metadata, resolve_missing_key_identity=resolve_missing_key_identity
+        )
 
     @staticmethod
     async def _enrich_failure_metadata_with_key_info(metadata: dict, resolve_missing_key_identity: bool = True) -> dict:

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -64,11 +64,16 @@ from litellm.llms.azure.passthrough.transformation import (
 from litellm.llms.bedrock.passthrough.transformation import is_bedrock_streaming_endpoint
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.llms.deepgram.common_utils import (
+    DeepgramSpeakMeter,
     deepgram_listen_callback_params,
     deepgram_listen_is_priced,
     deepgram_listen_registry_key,
     deepgram_listen_requested_model,
     deepgram_listen_websocket_target,
+    deepgram_speak_character_rate,
+    deepgram_speak_registry_key,
+    deepgram_speak_requested_model,
+    deepgram_speak_websocket_target,
 )
 from litellm.llms.fal_ai.cost_calculator import fal_ai_passthrough_cost, fal_ai_queue_base
 from litellm.llms.nvidia_nim.passthrough.transformation import nvidia_nim_model_group_in_path
@@ -129,6 +134,7 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
     YieldsToConfiguredPassThroughRoute,
     create_pass_through_route,
     create_websocket_passthrough_route,
+    truncated_close_reason,
     websocket_passthrough_request,
 )
 from litellm.proxy.utils import ProxyLogging as ProxyLoggingType
@@ -145,6 +151,7 @@ from litellm.types.passthrough_endpoints.pass_through_endpoints import (
     LITELLM_PASS_THROUGH_CUSTOM_BODY_STATE_KEY,
     LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY,
     LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY,
+    WebSocketSessionMeter,
 )
 from litellm.types.passthrough_endpoints.tinyfish import (
     TINYFISH_AUTHENTICATED_RUN_FIELDS,
@@ -3460,6 +3467,7 @@ class _WebsocketRelay(Protocol):
         forward_headers: bool,
         endpoint: str,
         accept_websocket: bool,
+        meter: WebSocketSessionMeter | None = None,
     ) -> None: ...
 
 
@@ -3621,6 +3629,54 @@ async def deepgram_listen_websocket_route(
     )
 
 
+async def deepgram_speak_user_api_key_auth(websocket: WebSocket) -> UserAPIKeyAuth:
+    return await user_api_key_auth_websocket_for_model(
+        websocket, model=deepgram_speak_requested_model(websocket.url.query)
+    )
+
+
+@router.websocket("/deepgram/v1/speak")
+@router.websocket("/deepgram/speak")
+async def deepgram_speak_websocket_route(
+    websocket: WebSocket,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(deepgram_speak_user_api_key_auth)],
+    relay: Annotated[_WebsocketRelay, Depends(_websocket_relay)],
+) -> None:
+    deepgram_api_key: Final = passthrough_endpoint_router.get_credentials(
+        custom_llm_provider=litellm.LlmProviders.DEEPGRAM.value,
+        region_name=None,
+    )
+    if deepgram_api_key is None:
+        await websocket.close(code=1011, reason=_DEEPGRAM_WS_MISSING_KEY_REASON)
+        return
+
+    await websocket.accept(subprotocol=_negotiated_websocket_subprotocol(websocket))
+    target: Final = deepgram_speak_websocket_target(
+        api_base=get_secret_str("DEEPGRAM_API_BASE"),
+        query_string=websocket.url.query,
+    )
+    character_rate: Final = deepgram_speak_character_rate(target)
+    if character_rate is None:
+        await websocket.close(
+            code=1008,
+            reason=truncated_close_reason(
+                _DEEPGRAM_WS_UNPRICED_REASON.format(registry_key=deepgram_speak_registry_key(target))
+            ),
+        )
+        return
+
+    await relay(
+        websocket=websocket,
+        target=target,
+        custom_headers={"Authorization": f"Token {deepgram_api_key}"},
+        user_api_key_dict=user_api_key_dict,
+        forward_headers=False,
+        endpoint=websocket.url.path,
+        accept_websocket=False,
+        meter=DeepgramSpeakMeter(target, character_rate),
+    )
+
+
 _OPENAI_BETA_HEADER: Final = "openai-beta"
 _CLIENT_OPENAI_BETA_RESOURCES: Final = frozenset({"agents", "vaults", "chatkit"})
 
@@ -4474,10 +4530,16 @@ async def watsonx_proxy_route(
     if provider_config is None:
         raise HTTPException(status_code=404, detail="Watsonx passthrough config not found")
 
+    watsonx_credentials: Final = passthrough_endpoint_router.get_proxy_wide_deployment_credentials(
+        custom_llm_provider="watsonx"
+    )
+    api_key: Final = watsonx_credentials.api_key if watsonx_credentials is not None else None
+    api_base: Final = watsonx_credentials.api_base if watsonx_credentials is not None else None
+
     # Get complete URL with version parameter
     complete_url, _ = provider_config.get_complete_url(
-        api_base=None,
-        api_key=None,
+        api_base=api_base,
+        api_key=api_key,
         model="",
         endpoint=endpoint,
         request_query_params=None,
@@ -4491,8 +4553,8 @@ async def watsonx_proxy_route(
         messages=[],
         optional_params={},
         litellm_params={},
-        api_key=None,
-        api_base=None,
+        api_key=api_key,
+        api_base=api_base,
     )
 
     # Check for streaming

litellm/proxy/pass_through_endpoints/llm_provider_handlers/openai_passthrough_logging_handler.py

@@ -33,6 +33,7 @@ from litellm.proxy.pass_through_endpoints.llm_provider_handlers.base_passthrough
 from litellm.proxy.pass_through_endpoints.success_handler import (
     PassThroughEndpointLogging,
 )
+from litellm.types.decisions import OpenAIDecisionResponse
 from litellm.types.llms.openai import ResponsesAPIResponse
 from litellm.types.passthrough_endpoints.pass_through_endpoints import (
     SETTLES_BACKGROUND_RESPONSE_KWARG,
@@ -277,6 +278,22 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
     def is_openai_agents_route(url_route: str) -> bool:
         return is_openai_compatible_url(url_route) and is_openai_agents_api_path(urlparse(url_route).path)
 
+    @staticmethod
+    def is_openai_decisions_route(url_route: str) -> bool:
+        return is_openai_compatible_url(url_route) and urlparse(url_route).path.rstrip("/").endswith("/v1/decisions")
+
+    @staticmethod
+    def handles_route(url_route: str) -> bool:
+        return (
+            OpenAIPassthroughLoggingHandler.is_openai_chat_completions_route(url_route)
+            or OpenAIPassthroughLoggingHandler.is_openai_embeddings_route(url_route)
+            or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route)
+            or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route)
+            or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route)
+            or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route)
+            or OpenAIPassthroughLoggingHandler.is_openai_decisions_route(url_route)
+        )
+
     @staticmethod
     def _unbilled_call_model(
         url_route: str, request_body: Mapping[str, object], response_body: Mapping[str, object]
@@ -479,8 +496,19 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
             httpx_response.request.method, url_route, response_body
         )
         is_responses: Final = responses_billing != "unbilled"
+        is_decisions: Final = (
+            httpx_response.request.method == "POST"
+            and OpenAIPassthroughLoggingHandler.is_openai_decisions_route(url_route)
+        )
 
-        if not (is_chat_completions or is_embeddings or is_image_generation or is_image_editing or is_responses):
+        if not (
+            is_chat_completions
+            or is_embeddings
+            or is_image_generation
+            or is_image_editing
+            or is_responses
+            or is_decisions
+        ):
             OpenAIPassthroughLoggingHandler._label_unbilled_call(
                 logging_obj,
                 url_route,
@@ -512,7 +540,13 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
         try:
             response_cost = 0.0
             litellm_model_response: (
-                ModelResponse | TextCompletionResponse | EmbeddingResponse | ImageResponse | ResponsesAPIResponse | None
+                ModelResponse
+                | TextCompletionResponse
+                | EmbeddingResponse
+                | ImageResponse
+                | ResponsesAPIResponse
+                | OpenAIDecisionResponse
+                | None
             ) = None
             handler_instance: Final = OpenAIPassthroughLoggingHandler()
 
@@ -564,11 +598,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
                     response_body=response_body,
                     request_body=request_body,
                 )
-                # Mark call type for downstream image-aware logic/metrics
-                try:
-                    logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
-                except Exception:
-                    pass
+                logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
                 # Create a simple response object for logging
                 litellm_model_response = ImageResponse(
                     data=response_body.get("data", []),
@@ -583,11 +613,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
                     response_body=response_body,
                     request_body=request_body,
                 )
-                # Mark call type for downstream image-aware logic/metrics
-                try:
-                    logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
-                except Exception:
-                    pass
+                logging_obj.call_type = PassthroughCallTypes.passthrough_image_generation.value
                 # Create a simple response object for logging
                 litellm_model_response = ImageResponse(
                     data=response_body.get("data", []),
@@ -609,6 +635,14 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
                     logging_obj=logging_obj,
                     custom_llm_provider=custom_llm_provider,
                 )
+            elif is_decisions:
+                litellm_model_response = OpenAIDecisionResponse.model_validate(response_body)  # rebind-ok: elif-chain
+                response_cost = litellm.completion_cost(  # rebind-ok: elif-chain
+                    completion_response=litellm_model_response,
+                    model=model,
+                    custom_llm_provider=custom_llm_provider,
+                    call_type="decisions",
+                )
 
             # Update kwargs with cost information
             kwargs["response_cost"] = response_cost
@@ -656,6 +690,8 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
                 if is_image_generation
                 else "image_editing"
                 if is_image_editing
+                else "decisions"
+                if is_decisions
                 else "responses"
             )
             verbose_proxy_logger.debug(

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -2584,7 +2584,7 @@ def _json_object_frame(frame: str | bytes) -> dict[str, object] | None:
     return decoded if isinstance(decoded, dict) else None
 
 
-def _truncated_close_reason(reason: str) -> str:
+def truncated_close_reason(reason: str) -> str:
     """
     Fit a close reason inside the byte budget a WebSocket close frame allows, without splitting a character
     """
@@ -2958,7 +2958,7 @@ async def websocket_passthrough_request(
             if upstream_close is not None and _client_socket_is_open(websocket):
                 await websocket.close(
                     code=upstream_close.code,
-                    reason=_truncated_close_reason(upstream_close.reason),
+                    reason=truncated_close_reason(upstream_close.reason),
                 )
 
             end_time: Final = datetime.now()

litellm/proxy/pass_through_endpoints/passthrough_endpoint_router.py

@@ -132,6 +132,19 @@ class PassthroughEndpointRouter:
             api_base=deployment_credentials.api_base if deployment_credentials is not None else None,
         )
 
+    def get_proxy_wide_deployment_credentials(self, custom_llm_provider: str) -> _PassThroughCredentials | None:
+        deployment_credentials: Final = self._get_deployment_credentials(
+            custom_llm_provider=custom_llm_provider,
+            region_name=None,
+            include_team_deployments=False,
+        )
+        if deployment_credentials is None:
+            return None
+        api_key: Final = _resolve_oidc_reference(deployment_credentials.api_key)
+        if not api_key:
+            return None
+        return _PassThroughCredentials(api_key=api_key, api_base=deployment_credentials.api_base)
+
     def has_credentials(self, custom_llm_provider: str) -> bool:
         if (
             self._get_deployment_credentials(

litellm/proxy/pass_through_endpoints/success_handler.py

@@ -666,28 +666,11 @@ class PassThroughEndpointLogging:
         return False
 
     def _is_supported_openai_endpoint(self, url_route: str) -> bool:
-        """Check if the OpenAI endpoint is supported by the passthrough logging handler.
-
-        The Responses API route is included because
-        `openai_passthrough_handler` has a dedicated `elif is_responses:`
-        branch that knows how to extract usage + cost from the
-        Responses-API on-the-wire shape. Without including it here, the
-        outer dispatch filters Responses calls out before reaching the
-        handler — the inner branch is then unreachable and Responses
-        calls land in `LiteLLM_SpendLogs` with zero tokens / zero spend.
-        """
         from .llm_provider_handlers.openai_passthrough_logging_handler import (
             OpenAIPassthroughLoggingHandler,
         )
 
-        return (
-            OpenAIPassthroughLoggingHandler.is_openai_chat_completions_route(url_route)
-            or OpenAIPassthroughLoggingHandler.is_openai_embeddings_route(url_route)
-            or OpenAIPassthroughLoggingHandler.is_openai_image_generation_route(url_route)
-            or OpenAIPassthroughLoggingHandler.is_openai_image_editing_route(url_route)
-            or OpenAIPassthroughLoggingHandler.is_openai_responses_route(url_route)
-            or OpenAIPassthroughLoggingHandler.is_openai_agents_route(url_route)
-        )
+        return OpenAIPassthroughLoggingHandler.handles_route(url_route)
 
     def _set_cost_per_request(
         self,

litellm/proxy/proxy_server.py

@@ -473,7 +473,6 @@ from litellm.proxy.common_request_processing import (  # noqa: F401, RUF100  # l
     log_llm_api_exception,
     open_sse_before_first_byte,
     proxy_exception_from_route_error,
-    request_litellm_call_id,
     resolve_litellm_call_id,
     should_return_raw_model_name,
     sse_error_payload,
@@ -840,11 +839,15 @@ from litellm.proxy.spend_tracking.spend_event_producer import (
     SpendEventProducer,
     build_spend_event_producer,
 )
+from litellm.proxy.spend_tracking.usage_capture import UsageBatchWriter, UsageCaptureRuntime
+from litellm.proxy.spend_tracking.usage_meter import UsageCaptureSettings
 from litellm.proxy.telemetry.endpoints import router as telemetry_router
 from litellm.proxy.telemetry.middleware import TelemetryMiddleware
 from litellm.proxy.telemetry.runtime import TelemetryRuntime
 from litellm.proxy.telemetry.settings import describe_errors as describe_telemetry_errors
 from litellm.proxy.telemetry.settings import load_settings as load_telemetry_settings
+from litellm.router import router_arguments
+from litellm.router_config import build_router_config
 
 try:
     from litellm.proxy.enterprise_billing.billing_metrics import (
@@ -1329,6 +1332,7 @@ async def _flush_spend_around_logging_drain_on_shutdown(
     finally:
         await worker.stop(discard_pending=True, timeout=GracefulShutdownManager.remaining(2.0, clock=clock))
     await drain_tasks(GracefulShutdownManager.remaining(5.0, clock=clock))
+    await usage_capture.stop(GracefulShutdownManager.remaining(clock=clock))
     await flush_spend()
 
 
@@ -3202,9 +3206,39 @@ def load_from_azure_key_vault(use_azure_key_vault: bool = False):
 
 spend_event_producer: SpendEventProducer | None = None
 
+usage_capture: Final = UsageCaptureRuntime()
+
+
+def _configure_usage_capture() -> None:
+    settings: Final = _GENERAL_SETTINGS_VIEW.validate_python(general_settings)
+    capture_settings: Final = settings.get("accounting_capture")
+    if settings.get("accounting_sync_ack") and capture_settings is None:
+        raise ValueError("accounting_sync_ack requires accounting_capture")
+    if capture_settings is None:
+        if usage_capture.meter is not None:
+            raise ValueError("Disabling accounting_capture requires a worker restart")
+        return
+    if (
+        prisma_client is None
+        or CollectorSettings().enabled
+        or os.getenv("SPEND_LOGS_URL")
+        or litellm.apply_search_free_tier
+    ):
+        raise ValueError(
+            "Usage capture requires local Prisma writes and disabled collector, remote writer and search free tier"
+        )
+    from litellm.proxy.db.accounting_ledger import AccountingLedger
+
+    ledger: Final = AccountingLedger.for_client(prisma_client)
+    writer: Final = UsageBatchWriter(ledger.insert_usage_batch, ledger.load_usage_batch)
+    usage_capture.start(
+        writer.write, UsageCaptureSettings.model_validate(capture_settings), load=ledger.load_usage_batch
+    )
+
 
 def cost_tracking():
     global prisma_client, spend_event_producer
+    _configure_usage_capture()
     if prisma_client is not None:
         from litellm.integrations.shadow_eval_logger import ShadowEvalLogger
 
@@ -7375,20 +7409,24 @@ class ProxyConfig:
                     verbose_proxy_logger.warning(
                         "Key '%s' is not a valid argument for Router.__init__(). Ignoring this key.", k
                     )
-        router = litellm.Router(
-            **router_params,
-            assistants_config=assistants_config,
-            search_tools=search_tools,
-            router_general_settings=RouterGeneralSettings(
-                async_only_mode=True  # only init async clients
-            ),
-            ignore_invalid_deployments=True,  # don't raise an error if a deployment is invalid
-            fallback_access_check=router_fallback_access_check,
-            fallback_budget_check=RouterFallbackBudgetCheck(
-                is_enforced=router_fallback_budget_check.is_enforced,
-                team_member_model_budget_limiter=model_max_budget_limiter,
-            ),
-            auto_router_capability_limit=_license_check.auto_router_capability_limit,
+        router = litellm.Router.from_config(
+            build_router_config(
+                router_arguments(
+                    **router_params,
+                    assistants_config=assistants_config,
+                    search_tools=search_tools,
+                    router_general_settings=RouterGeneralSettings(
+                        async_only_mode=True  # only init async clients
+                    ),
+                    ignore_invalid_deployments=True,  # don't raise an error if a deployment is invalid
+                    fallback_access_check=router_fallback_access_check,
+                    fallback_budget_check=RouterFallbackBudgetCheck(
+                        is_enforced=router_fallback_budget_check.is_enforced,
+                        team_member_model_budget_limiter=model_max_budget_limiter,
+                    ),
+                    auto_router_capability_limit=_license_check.auto_router_capability_limit,
+                )
+            )
         )
 
         if redis_usage_cache is not None and router.health().store.redis_cache is None:
@@ -7878,19 +7916,23 @@ class ProxyConfig:
                 # Router can function with model_list=[] if search_tools are configured
                 if len(_model_list) > 0 or search_tools:
                     verbose_proxy_logger.debug("_model_list: %s", _model_list)
-                    llm_router = litellm.Router(
-                        model_list=_model_list,
-                        router_general_settings=RouterGeneralSettings(
-                            async_only_mode=True  # only init async clients
-                        ),
-                        search_tools=search_tools,
-                        ignore_invalid_deployments=True,
-                        fallback_access_check=router_fallback_access_check,
-                        fallback_budget_check=RouterFallbackBudgetCheck(
-                            is_enforced=router_fallback_budget_check.is_enforced,
-                            team_member_model_budget_limiter=model_max_budget_limiter,
-                        ),
-                        auto_router_capability_limit=_license_check.auto_router_capability_limit,
+                    llm_router = litellm.Router.from_config(
+                        build_router_config(
+                            router_arguments(
+                                model_list=_model_list,
+                                router_general_settings=RouterGeneralSettings(
+                                    async_only_mode=True  # only init async clients
+                                ),
+                                search_tools=search_tools,
+                                ignore_invalid_deployments=True,
+                                fallback_access_check=router_fallback_access_check,
+                                fallback_budget_check=RouterFallbackBudgetCheck(
+                                    is_enforced=router_fallback_budget_check.is_enforced,
+                                    team_member_model_budget_limiter=model_max_budget_limiter,
+                                ),
+                                auto_router_capability_limit=_license_check.auto_router_capability_limit,
+                            )
+                        )
                     )
                     verbose_proxy_logger.debug("updated llm_router: %s", llm_router)
             else:
@@ -13006,8 +13048,10 @@ async def completion(
     """
     global user_temperature, user_request_timeout, user_max_tokens, user_api_base
     data = {}
+    base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
     try:
         data = await read_request_body(request=request)  # rebind-ok: pre-existing rebinding on a rename-only line
+        base_llm_response_processor.data = data
         if user_api_key_dict is not None:
             if data.get("metadata") is None:
                 data["metadata"] = {}
@@ -13021,7 +13065,6 @@ async def completion(
                 data["metadata"]["user_api_key_org_alias"] = user_api_key_dict.organization_alias
             if hasattr(user_api_key_dict, "agent_id") and user_api_key_dict.agent_id is not None:
                 data["metadata"]["agent_id"] = user_api_key_dict.agent_id
-        base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data)
         return await base_llm_response_processor.base_process_llm_request(
             request=request,
             fastapi_response=fastapi_response,
@@ -13123,9 +13166,12 @@ async def completion(
             return _response
     except Exception as e:
         await proxy_logging_obj.post_call_failure_hook(
-            user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data
+            user_api_key_dict=user_api_key_dict,
+            original_exception=e,
+            request_data=base_llm_response_processor.data,
+            dispatched_call_type=base_llm_response_processor.route_type,
         )
-        litellm_call_id: Final = request_litellm_call_id(data)
+        litellm_call_id: Final = base_llm_response_processor.litellm_call_id
         log_llm_api_exception(e, litellm_call_id)
         if isinstance(e, ProxyException):
             raise with_litellm_call_id(e, litellm_call_id)

litellm/proxy/route_llm_request.py

@@ -18,7 +18,9 @@ from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.routing_common_utils import is_proxy_admin_request
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
+from litellm.router import router_arguments
 from litellm.router_api import router_init_args
+from litellm.router_config import build_router_config
 
 # Client-supplied params that make the router or the call path fabricate a
 # failure or a delay instead of calling the provider. The ``mock_testing_*``
@@ -98,7 +100,9 @@ async def _route_user_config_request(data: dict, route_type: str):
     filtered_config: Final = {k: v for k, v in router_config.items() if k in valid_args}
 
     logging_obj: Final = _request_logging_obj(JSON_OBJECT_ADAPTER.validate_python(data), route_type)
-    cached_router: Final = user_config_routers.get(filtered_config, lambda: litellm.Router(**filtered_config))
+    cached_router: Final = user_config_routers.get(
+        filtered_config, lambda: litellm.Router.from_config(build_router_config(router_arguments(**filtered_config)))
+    )
     cached_router.observe(logging_obj)
     return await getattr(cached_router, f"{route_type}")(**{**data, "litellm_logging_obj": logging_obj})
 

litellm/proxy/spend_tracking/cost_finalization.py

@@ -89,6 +89,7 @@ async def _finish_finalization(
             if litellm.apply_search_free_tier
             else None
         )
+        logging_obj.cost_allowance_allocation = allocation  # rebind-ok: retain the allocation for durable capture
         logging_obj.cost_allowance_discount = (  # rebind-ok: retain allocation across snapshot retries
             allocation.discount if allocation is not None else 0.0
         )

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -3,7 +3,7 @@ import json
 import os
 import re
 import secrets
-from collections.abc import Mapping, Sequence
+from collections.abc import Callable, Mapping, Sequence
 from datetime import datetime, timezone
 from datetime import datetime as dt
 from functools import reduce
@@ -55,6 +55,8 @@ from litellm.litellm_core_utils.get_llm_provider_logic import declared_authentic
 from litellm.litellm_core_utils.hidden_params import served_from_cache
 from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call
 from litellm.litellm_core_utils.litellm_logging import (
+    StandardLoggingPayloadSetup,
+    UsageObservation,
     coerce_model_access_groups,
     is_valid_sha256_hash,
     request_model_access_groups_from_litellm_params,
@@ -599,6 +601,7 @@ def get_logging_payload(
     start_time: datetime,
     end_time: datetime,
     llm_router: "Router | None" = None,
+    capture_usage: Callable[[SpendLogsPayload, str | None], None] | None = None,
 ) -> SpendLogsPayload:
     if kwargs is None:
         kwargs = {}
@@ -622,7 +625,9 @@ def get_logging_payload(
         routing_metadata,
         routing_litellm_params,
     )
-    completion_start_time: Final = kwargs.get("completion_start_time", end_time)
+    completion_start_time: Final = TypeAdapter[datetime | None](datetime | None).validate_python(
+        kwargs.get("completion_start_time", end_time)
+    )
     call_type: Final = kwargs.get("call_type")
     cache_hit: Final = kwargs.get("cache_hit", False)
 
@@ -644,7 +649,7 @@ def get_logging_payload(
         _usage: Final = (
             chat_usage_of_batch.model_dump()
             if chat_usage_of_batch is not None
-            else response_obj_dict.get("usage", None) or {}
+            else StandardLoggingPayloadSetup.get_usage_as_dict(response_obj_dict)
         )
         if isinstance(_usage, litellm.Usage):
             usage = dict(_usage)
@@ -928,7 +933,9 @@ def get_logging_payload(
             cache_hit=str(cache_hit),
             startTime=_ensure_datetime_utc(start_time),
             endTime=_ensure_datetime_utc(end_time),
-            completionStartTime=_ensure_datetime_utc(completion_start_time),
+            completionStartTime=(
+                _ensure_datetime_utc(completion_start_time) if completion_start_time is not None else None
+            ),
             model=model_name,
             user=metadata.get("user_api_key_user_id", "") or "",
             team_id=metadata.get("user_api_key_team_id", "") or "",
@@ -984,6 +991,14 @@ def get_logging_payload(
             payload.get("spend"),
         )
 
+        if capture_usage is not None:
+            observed_usage: Final = (
+                StandardLoggingPayloadSetup.get_usage_as_dict({"usage": usage})
+                if usage and kwargs.get("_accounting_usage_observed") is not UsageObservation.UNKNOWN
+                else None
+            )
+            capture_usage(payload, safe_dumps(observed_usage) if observed_usage is not None else None)
+
         # Explicitly clear large intermediate objects to reduce memory pressure
         del response_obj_dict, usage, clean_metadata, additional_usage_values
 

litellm/proxy/spend_tracking/usage_meter.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import asyncio
 from collections import deque
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Mapping
 from concurrent.futures import Future
 from dataclasses import dataclass
 from itertools import accumulate, chain, count, takewhile
@@ -114,6 +114,9 @@ class DurableUsageMeter:
         self._admission: Final = Lock()
         self._queue: Final[deque[_Pending]] = deque()  # mutable-ok: one bounded producer/consumer queue
         self._wakeup: Final = asyncio.Event()
+        self._capacity_changed: Final = asyncio.Event()
+        self._reservations: Mapping[object, tuple[int, int]] = {}
+        self._reserved_bytes = 0
         self._task: asyncio.Task[None] | None = None
         self._closed = False
         self.pending_bytes = 0
@@ -143,6 +146,36 @@ class DurableUsageMeter:
         wrapped.add_done_callback(lambda done: None if done.cancelled() else done.exception())
         return wrapped
 
+    def reserve_producer(self, size: int, headroom: int) -> object:
+        token: Final = object()
+        self.resize_producer(token, size, headroom)
+        return token
+
+    def resize_producer(self, token: object, size: int, headroom: int) -> None:
+        with self._admission:
+            previous: Final = self._reservations.get(token, (0, 0))[0]
+            delta: Final = size - previous
+            others: Final = (value[1] for key, value in self._reservations.items() if key is not token)
+            required: Final = max((headroom, *others))
+            if (
+                self._closed
+                or self.pending_bytes + delta > self.settings.max_pending_bytes
+                or self._reserved_bytes + delta + required > self.settings.max_pending_bytes
+            ):
+                raise UsageCaptureError("Batch capture producer byte capacity exhausted")
+            self._reservations = {**self._reservations, token: (size, headroom)}
+            self._reserved_bytes += delta
+            self.pending_bytes += delta
+        self._capacity_changed.set()
+
+    def release_producer(self, token: object) -> None:
+        with self._admission:
+            size, _ = self._reservations[token]
+            self._reservations = {key: value for key, value in self._reservations.items() if key is not token}
+            self._reserved_bytes -= size
+            self.pending_bytes -= size
+        self._capacity_changed.set()
+
     def _ensure_worker(self) -> None:
         if self.pending_records >= self.settings.batch_size or self._queue and self._queue[0].prerequisites:
             self._wakeup.set()
@@ -153,6 +186,34 @@ class DurableUsageMeter:
         envelope: Final = '{"event":' + event.model_dump_json() + "}"
         await asyncio.shield(self.submit((UsageWrite(event, envelope, usage_delivery_targets(event.dimensions)),)))
 
+    async def record_writes(self, writes: tuple[UsageWrite, ...]) -> None:
+        size: Final = sum(len(write.envelope.encode("utf-8")) for write in writes)
+        capacity: Final = self.settings.max_pending_bytes - self._reserved_bytes
+        if len(writes) > min(self.settings.batch_size, self.settings.max_pending_records) or (
+            size > capacity and len(writes) > 1
+        ):
+            middle: Final = len(writes) // 2
+            await self.record_writes(writes[:middle])
+            await self.record_writes(writes[middle:])
+            return
+        if size > capacity:
+            raise UsageCaptureError("One usage observation exceeds capture capacity")
+        while (receipt := self._submit_available(writes)) is None:
+            self._wakeup.set()
+            await self._capacity_changed.wait()
+        self._wakeup.set()
+        await asyncio.shield(receipt)
+
+    def _submit_available(self, writes: tuple[UsageWrite, ...]) -> asyncio.Future[None] | None:
+        self._capacity_changed.clear()
+        try:
+            receipt: Final = self.submit(writes)
+        except UsageCaptureError:
+            if self._closed or not self.pending_records or not writes:
+                raise
+            return None
+        return receipt
+
     async def flush(self) -> None:
         self._ensure_worker()
         self._wakeup.set()
@@ -240,6 +301,7 @@ class DurableUsageMeter:
             with self._admission:
                 self.pending_bytes -= item.size
                 self.pending_records -= len(item.writes)
+            self._capacity_changed.set()
             if item.committed.done():
                 continue
             if error is None:

litellm/proxy/utils.py

@@ -163,7 +163,12 @@ from litellm.litellm_core_utils.core_helpers import (
 from litellm.litellm_core_utils.deployment_pricing import pricing_models
 from litellm.litellm_core_utils.duration_parser import get_budget_window_start
 from litellm.litellm_core_utils.env_utils import get_env_float, get_env_int
-from litellm.litellm_core_utils.litellm_logging import BilledRoundsDispatch, Logging, deployment_pricing_model_info
+from litellm.litellm_core_utils.litellm_logging import (
+    BilledRoundsDispatch,
+    Logging,
+    UsageObservation,
+    deployment_pricing_model_info,
+)
 from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
 from litellm.litellm_core_utils.routing_common_utils import resolve_model_group_alias
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@@ -1739,6 +1744,39 @@ def _failure_fields_to_lift(
         if _prompt_estimate is not None
         else ()
     )
+    from litellm.litellm_core_utils.litellm_logging import UsageCaptureObservation
+
+    _recorded_observation: Final[object] = _model_call_details.get("_accounting_usage_observed")
+    _current_usage: Final = (
+        _current_failure_usage_to_lift(
+            _model_call_details,
+            request_data,
+            _model_call_details.get("current_attempt_api_call_start_time", _first_handoff) is not None,
+            _billed,
+            _deployment_id,
+            _base_model,
+            dispatched_call_type,
+        )
+        if Logging.usage_capture_callback is not None
+        and (_billed or _model_call_details.get("combined_usage_object") is not None)
+        else None
+    )
+    _current_observation: Final = (
+        _recorded_observation
+        if isinstance(_recorded_observation, UsageObservation)
+        else UsageObservation.UNKNOWN
+        if _current_usage is None
+        else UsageObservation.KNOWN
+        if _model_call_details.get("combined_usage_object") is not None
+        else UsageObservation.ESTIMATED
+    )
+    _capture_observation: Final = UsageCaptureObservation.from_usage(
+        _current_usage[0] if _current_usage is not None and isinstance(_current_usage[0], Usage) else None,
+        _current_observation,
+        float(_current_usage[1])
+        if _current_usage is not None and isinstance(_current_usage[1], (int, float))
+        else None,
+    )
     _entries: Final = (
         ("start_time", _model_call_details.get("start_time")),
         ("first_api_call_start_time", _first_handoff),
@@ -1747,6 +1785,16 @@ def _failure_fields_to_lift(
         ("response_cost", None if _usage_to_lift is None else (_usage_to_lift[1] or 0.0)),
         ("standard_logging_object", _model_call_details.get("standard_logging_object")),
         ("billed_failed_attempts", _earlier_attempts or None),
+        ("_accounting_capture", _model_call_details.get("_accounting_capture")),
+        ("_accounting_pricing_snapshot", _model_call_details.get("_accounting_pricing_snapshot")),
+        ("_accounting_finalized_cost", _logging_obj.finalized_cost if isinstance(_logging_obj, Logging) else None),
+        (
+            "_accounting_allocation",
+            _logging_obj.cost_allowance_allocation if isinstance(_logging_obj, Logging) else None,
+        ),
+        ("_accounting_usage_observed", _current_observation if Logging.usage_capture_callback is not None else None),
+        ("_accounting_usage_snapshot", _capture_observation if Logging.usage_capture_callback is not None else None),
+        ("_accounting_prior_captures", _model_call_details.get("_accounting_prior_captures")),
     )
     return MappingProxyType({key: value for key, value in _entries if value is not None})
 

litellm/rag/main.py

@@ -223,6 +223,10 @@ async def _execute_query_pipeline(
     """
     Execute the RAG query pipeline.
     """
+    parent_request_id: Final[str | None] = TypeAdapter[str | None](str | None).validate_python(
+        TypeAdapter(dict[str, object]).validate_python(kwargs).get("litellm_call_id")
+    )
+
     # Extract router from kwargs - use it for completion if available
     # to properly resolve virtual model names
     router: Final[Router | None] = kwargs.pop("router", None)
@@ -252,7 +256,7 @@ async def _execute_query_pipeline(
     forwarded_search_params: Final = MappingProxyType(
         {**provider_search_params, **kwargs, **filter_search_params, **store_search_params}
     )
-    with internal_sub_call():
+    with internal_sub_call(parent_request_id):
         search_response: Final = await litellm.vector_stores.asearch(
             vector_store_id=retrieval_config["vector_store_id"],
             query=query_text,
@@ -286,7 +290,7 @@ async def _execute_query_pipeline(
     if rerank and rerank.get("enabled"):
         documents: Final = RAGQuery.extract_documents_from_search(search_response)
         if documents:
-            with internal_sub_call():
+            with internal_sub_call(parent_request_id):
                 rerank_response = await litellm.arerank(
                     model=rerank["model"],
                     query=query_text,
@@ -304,7 +308,7 @@ async def _execute_query_pipeline(
     modified_messages: Final = messages[:-1] + [context_message] + [messages[-1]]
 
     # Use router if available to properly resolve virtual model names
-    with internal_sub_call():
+    with internal_sub_call(parent_request_id):
         if router is not None:
             response = await router.acompletion(
                 model=model,

litellm/responses/main.py

@@ -437,7 +437,7 @@ async def aresponses_api_with_mcp(
     #########################################################
     if isinstance(logging_obj, LiteLLMLoggingObj):
         logging_obj.carry_failed_attempt_usage()
-    with internal_sub_call():
+    with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
         response: Final = await aresponses(
             input=input,
             model=model,
@@ -514,7 +514,9 @@ async def aresponses_api_with_mcp(
                             tool_calls=tool_calls, tool_results=tool_results
                         )
 
-                    with internal_sub_call():
+                    with internal_sub_call(
+                        logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None
+                    ):
                         final_response = await LiteLLM_Proxy_MCP_Handler.make_follow_up_call(
                             follow_up_input=follow_up_input,
                             model=model,

litellm/responses/mcp/chat_completions_handler.py

@@ -663,7 +663,7 @@ async def acompletion_with_mcp(
     logging_obj: Final = kwargs.get("litellm_logging_obj")
     if isinstance(logging_obj, LiteLLMLoggingObj):
         logging_obj.carry_failed_attempt_usage()
-    with internal_sub_call():
+    with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
         initial_response: Final = await litellm_acompletion(**initial_call_args)
     first_round_breakdown: Final = logging_obj.cost_breakdown if isinstance(logging_obj, LiteLLMLoggingObj) else None
 
@@ -713,7 +713,7 @@ async def acompletion_with_mcp(
         follow_up_call_args["messages"] = follow_up_messages
         follow_up_call_args["stream"] = stream
 
-        with internal_sub_call():
+        with internal_sub_call(logging_obj.litellm_call_id if isinstance(logging_obj, LiteLLMLoggingObj) else None):
             final_response: Final = await litellm_acompletion(**follow_up_call_args)
 
     if not isinstance(final_response, ModelResponse):

litellm/responses/streaming_iterator.py

@@ -41,6 +41,7 @@ from litellm.constants import (
 from litellm.exceptions import MidStreamFallbackError, RateLimitError, ServiceUnavailableError
 from litellm.litellm_core_utils.asyncify import run_async_function
 from litellm.litellm_core_utils.core_helpers import (
+    UPSTREAM_USAGE_COST,
     process_response_headers,
     provider_reported_usage_cost,
     set_response_cost_in_hidden_params,
@@ -184,6 +185,7 @@ def _typed_gets_litellm_params(fn: _GetsLitellmParams) -> _GetsLitellmParams:
 _SHOULD_STORE_RESULT_IN_CACHE_ATTR: Final = "_should_store_result_in_cache"
 _UNMASK_PII_TEXT_ATTR: Final = "_unmask_pii_text"
 _DECODED_JSON: Final = TypeAdapter(object)
+_HIDDEN_PARAMS: Final = TypeAdapter(dict[str, object])
 
 
 def _load_json_object(payload: str | bytes) -> dict[str, object]:
@@ -231,6 +233,11 @@ def spawn_logging_task(
 
 
 async def drain_logging_tasks(timeout: float = 5.0) -> None:
+    # A task that finished as its loop closed never ran its discard callback, and a
+    # task from another loop cannot be awaited here: forget both, wait on the rest.
+    running_loop: Final = asyncio.get_running_loop()
+    stale: Final = {task for task in _PENDING_LOGGING_TASKS if task.done() or task.get_loop() is not running_loop}
+    _PENDING_LOGGING_TASKS.difference_update(stale)
     if not _PENDING_LOGGING_TASKS:
         return
     _, pending = await asyncio.wait(tuple(_PENDING_LOGGING_TASKS), timeout=timeout)
@@ -693,7 +700,9 @@ class BaseResponsesAPIStreamingIterator:
         source_hidden: Final[object] = getattr(
             getattr(self.completed_response, "response", None), "_hidden_params", None
         )
-        source: Final[Mapping[str, object]] = source_hidden if isinstance(source_hidden, Mapping) else EMPTY_MAPPING
+        source: Final[Mapping[str, object]] = (
+            _HIDDEN_PARAMS.validate_python(source_hidden) if isinstance(source_hidden, Mapping) else {}
+        )
         processed: Final[object] = source.get("additional_headers") or self._hidden_params.get("additional_headers")
         raw: Final[object] = source.get("headers") or self._raw_response_headers
         headers: Final[Mapping[str, object]] = processed if isinstance(processed, Mapping) else EMPTY_MAPPING
@@ -703,6 +712,7 @@ class BaseResponsesAPIStreamingIterator:
         target.hidden_params = {
             "additional_headers": {**headers},
             "headers": {**raw_headers},
+            **_stamped_bill_hidden_params(source),
             **existing,
         }
 
@@ -1749,6 +1759,11 @@ def _dump_json_safely(response: BaseModel) -> str | None:
         return None
 
 
+def _stamped_bill_hidden_params(source: Mapping[str, object]) -> Mapping[str, object]:
+    stamped: Final = UPSTREAM_USAGE_COST in source
+    return {key: source[key] for key in (UPSTREAM_USAGE_COST, "response_cost") if stamped and key in source}
+
+
 def _logging_copy(event: object) -> object:
     """Hand logging callbacks a copy, so their usage rewrite (Responses shape to chat shape) never
     reaches the event the caller is iterating. The round trip through ``model_dump`` sidesteps the

litellm/router_api.py

@@ -42,6 +42,8 @@ from dataclasses import dataclass, field
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias, overload
 
+from litellm.router_config import detached
+
 if TYPE_CHECKING:
     from typing import Any  # noqa: TID251  # only to spell the cores' untyped (Unknown) containers and one Any type
 
@@ -176,9 +178,9 @@ def deployment_usable_by_team(deployment: "Mapping[str, object] | Deployment", t
     Whether ``team_id`` may use ``deployment``: a team-scoped deployment (``model_info.team_id``
     set) only by that team, any other deployment by everyone. Needs no router instance.
     """
-    from litellm.router import Router
+    from litellm.router_python_core import PythonCore
 
-    return Router._core_deployment_usable_by_team(deployment, team_id)
+    return PythonCore._core_deployment_usable_by_team(deployment, team_id)
 
 
 def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str:
@@ -200,9 +202,9 @@ def deployment_id(model_group: str, litellm_params: Mapping[str, object]) -> str
 
 def router_init_args() -> tuple[str, ...]:
     """The arguments ``Router.__init__`` accepts (was ``Router.get_valid_args``). Needs no router instance."""
-    from litellm.router import Router
+    from litellm.router_python_core import PythonCore
 
-    return tuple(Router.get_valid_args())
+    return tuple(PythonCore.get_valid_args())
 
 
 def mark_advisor_orchestration_failure(exception: BaseException) -> None:
@@ -266,9 +268,9 @@ def deployment_metadata(request_kwargs: Mapping[str, object]) -> Mapping[str, ob
 
 
 def _json_default_stable_id(obj: object) -> str:
-    from litellm.router import Router
+    from litellm.router_python_core import PythonCore
 
-    return Router._json_default_stable_id(obj)
+    return PythonCore._json_default_stable_id(obj)
 
 
 class RouterRegistry:
@@ -529,10 +531,10 @@ class RouterRegistry:
         ``should_include_deployment``), or, without ``model_name``, whether the team may use it at
         all (was ``deployment_usable_by_team``).
         """
-        from litellm.router import Router
+        from litellm.router_python_core import PythonCore
 
         if model_name is None:
-            return Router._core_deployment_usable_by_team(deployment, team_id)
+            return PythonCore._core_deployment_usable_by_team(deployment, team_id)
         return self._router._core_should_include_deployment(model_name=model_name, model=deployment, team_id=team_id)
 
     @overload
@@ -1053,6 +1055,12 @@ SETTING_FIELDS: Final[tuple[str, ...]] = (
     "cache_responses",
 )
 
+# Runtime settings kept as plain router attributes rather than ``settings()`` fields. A write
+# to one (``router.num_retries = 2``) is a changeset too: ``ChangeSet(assign={...})``.
+PLAIN_SETTING_ATTRIBUTES: Final[frozenset[str]] = frozenset({"num_retries", "max_fallbacks", "retry_after", "timeout"})
+# Former attributes whose value lives under another name, assignable through a changeset.
+ASSIGNABLE_ALIASES: Final[Mapping[str, str]] = MappingProxyType({"model_group_alias": "_core_model_group_alias"})
+
 _MISSING: Final = object()
 
 
@@ -1168,16 +1176,16 @@ class RouterHealth:
     @staticmethod
     def is_blocked(deployment: "Deployment") -> bool:
         """Whether an admin paused this deployment (was ``is_deployment_blocked``)."""
-        from litellm.router import Router
+        from litellm.router_python_core import PythonCore
 
-        return Router._core_is_deployment_blocked(deployment)
+        return PythonCore._core_is_deployment_blocked(deployment)
 
     @staticmethod
     def all_blocked(deployments: "Sequence[DeploymentTypedDict]") -> bool:
         """Whether every deployment in the list is paused (was ``are_all_deployments_blocked``)."""
-        from litellm.router import Router
+        from litellm.router_python_core import PythonCore
 
-        return Router._core_are_all_deployments_blocked(deployments)
+        return PythonCore._core_are_all_deployments_blocked(deployments)
 
     def fully_blocked_model_names(self) -> set[str]:  # mutable-ok: the core's type, a fresh set
         """Groups whose every deployment is paused (was ``get_fully_blocked_model_names``)."""
@@ -1296,13 +1304,17 @@ class RoutingObservers:
 def apply_changes(router: "Router", changes: ChangeSet) -> ApplyResult:
     if changes.refresh_model_info:
         raise TypeError("ChangeSet(refresh_model_info=True) needs aapply()")
-    unknown: Final = sorted(set(changes.assign) - set(SETTING_FIELDS))
+    unknown: Final = sorted(set(changes.assign) - {*SETTING_FIELDS, *PLAIN_SETTING_ATTRIBUTES, *ASSIGNABLE_ALIASES})
     if unknown:
         raise ValueError(f"not assignable router settings: {unknown}")
     for name, value in changes.assign.items():
-        setattr(router, "_core_" + name, value)
+        setattr(
+            router,
+            name if name in PLAIN_SETTING_ATTRIBUTES else ASSIGNABLE_ALIASES.get(name, "_core_" + name),
+            detached(value),
+        )
     if changes.settings:
-        router._core_update_settings(**changes.settings)
+        router._core_update_settings(**{name: detached(value) for name, value in changes.settings.items()})
     upserted: Final = [router._core_upsert_deployment(deployment=d) for d in changes.upsert]
     deleted: Final = [router._core_delete_deployment(id=i) for i in changes.delete]
     if changes.redis_cache is not None:
@@ -1388,7 +1400,7 @@ class RoutingContext:
         return self._router.observe()
 
     def belongs_to(self, router: "Router") -> bool:
-        return self._router is router
+        return self._router is router or router.__dict__.get("_core") is self._router
 
     @property
     def explicit_allowed_fails(self) -> int | None:

litellm/router_config.py

@@ -0,0 +1,157 @@
+"""
+The router's configuration, built outside the router.
+
+``RouterConfig`` is static: the constructor arguments, frozen for the router's life.
+``RouterDynamicConfig`` is a snapshot of the process settings the router reads while it serves
+(``litellm.num_retries``, ``litellm.model_cost``, the fallback lists, ...). A call reads one snapshot
+at its start. This module is the only router code that reads ``litellm.*`` settings.
+"""
+
+import itertools
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final
+
+import litellm
+
+if TYPE_CHECKING:
+    from litellm.caching.caching import Cache
+    from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
+
+
+@dataclass(frozen=True, slots=True)
+class RouterConfig:
+    """The arguments the router was built with, exactly as passed (unset ones are absent)."""
+
+    arguments: Mapping[str, object] = field(default_factory=lambda: MappingProxyType({}))
+
+    def get(self, name: str, default: object = None) -> object:
+        return self.arguments.get(name, default)
+
+
+@dataclass(frozen=True, slots=True)
+class RouterDynamicConfig:
+    """Process settings the router reads per call. Values are the live objects, not copies."""
+
+    version: int
+    num_retries: int | None
+    max_fallbacks: int | None
+    router_max_fallbacks: int
+    allowed_fails: int
+    request_timeout: float
+    fallbacks: Sequence[object] | None
+    default_fallbacks: Sequence[str] | None
+    context_window_fallbacks: Sequence[object] | None
+    content_policy_fallbacks: Sequence[object] | None
+    drop_params: bool | None
+    provider_list: Sequence[str]
+    expose_router_debug_in_errors: bool
+    redact_user_api_key_info: bool | None
+    model_alias_map: Mapping[str, str]
+    model_cost: Mapping[str, object]
+    known_custom_logger_compatible_callbacks: Sequence[str]
+    exception_types: Sequence[type[BaseException]]
+    callbacks: Sequence[object]
+    success_callback: Sequence[object]
+    failure_callback: Sequence[object]
+    input_callback: Sequence[object]
+    callback_manager: "LoggingCallbackManager"
+    cache: "Cache | None"
+
+
+_versions: Final = itertools.count(1)
+
+
+def detached(value: object) -> object:
+    """``value`` with its plain containers (dict, list, tuple, set) copied, all the way down, so the
+    caller's own objects can't change the router afterwards. Other objects (clients, caches,
+    callbacks, pydantic models) are kept as they are."""
+    if isinstance(value, dict):
+        return {key: detached(item) for key, item in value.items()}
+    if isinstance(value, list):
+        return [detached(item) for item in value]
+    if isinstance(value, tuple) and type(value) is tuple:
+        return tuple(detached(item) for item in value)
+    if isinstance(value, set):
+        return {detached(item) for item in value}
+    return value
+
+
+def build_router_config(arguments: Mapping[str, object]) -> RouterConfig:
+    """The constructor arguments, detached from the caller's objects: settings are static."""
+    return RouterConfig(arguments=MappingProxyType({name: detached(value) for name, value in arguments.items()}))
+
+
+def current_dynamic_config() -> RouterDynamicConfig:
+    """A snapshot of the process settings as they are right now."""
+    return RouterDynamicConfig(
+        version=next(_versions),
+        num_retries=litellm.num_retries,
+        max_fallbacks=litellm.max_fallbacks,
+        router_max_fallbacks=litellm.ROUTER_MAX_FALLBACKS,
+        allowed_fails=litellm.allowed_fails,
+        request_timeout=litellm.request_timeout,
+        fallbacks=litellm.fallbacks,
+        default_fallbacks=litellm.default_fallbacks,
+        context_window_fallbacks=litellm.context_window_fallbacks,
+        content_policy_fallbacks=litellm.content_policy_fallbacks,
+        drop_params=litellm.drop_params,
+        provider_list=litellm.provider_list,
+        expose_router_debug_in_errors=litellm.expose_router_debug_in_errors,
+        redact_user_api_key_info=litellm.redact_user_api_key_info,
+        model_alias_map=litellm.model_alias_map,
+        model_cost=litellm.model_cost,
+        known_custom_logger_compatible_callbacks=litellm._known_custom_logger_compatible_callbacks,
+        exception_types=litellm.LITELLM_EXCEPTION_TYPES,
+        callbacks=litellm.callbacks,
+        success_callback=litellm.success_callback,
+        failure_callback=litellm.failure_callback,
+        input_callback=litellm.input_callback,
+        callback_manager=litellm.logging_callback_manager,
+        cache=litellm.cache,
+    )
+
+
+_LIVE_SOURCES: Final = MappingProxyType(
+    {
+        "num_retries": "num_retries",
+        "max_fallbacks": "max_fallbacks",
+        "router_max_fallbacks": "ROUTER_MAX_FALLBACKS",
+        "allowed_fails": "allowed_fails",
+        "request_timeout": "request_timeout",
+        "fallbacks": "fallbacks",
+        "default_fallbacks": "default_fallbacks",
+        "context_window_fallbacks": "context_window_fallbacks",
+        "content_policy_fallbacks": "content_policy_fallbacks",
+        "drop_params": "drop_params",
+        "provider_list": "provider_list",
+        "expose_router_debug_in_errors": "expose_router_debug_in_errors",
+        "redact_user_api_key_info": "redact_user_api_key_info",
+        "model_alias_map": "model_alias_map",
+        "model_cost": "model_cost",
+        "known_custom_logger_compatible_callbacks": "_known_custom_logger_compatible_callbacks",
+        "exception_types": "LITELLM_EXCEPTION_TYPES",
+        "callbacks": "callbacks",
+        "success_callback": "success_callback",
+        "failure_callback": "failure_callback",
+        "input_callback": "input_callback",
+        "callback_manager": "logging_callback_manager",
+        "cache": "cache",
+    }
+)
+
+
+class LiveDynamicConfig:
+    """``RouterDynamicConfig``'s fields read from the process settings at access time, with no snapshot."""
+
+    __slots__ = ()
+
+    def __getattr__(self, name: str) -> object:
+        source: Final = _LIVE_SOURCES.get(name)
+        if source is None:
+            raise AttributeError(name)
+        return getattr(litellm, source)
+
+
+live_dynamic_config: Final = LiveDynamicConfig()

litellm/router_core.py

@@ -0,0 +1,56 @@
+"""
+``RouterCore``: the interface ``litellm.Router`` delegates to, and ``make_core``, which picks one.
+
+``PythonCore`` (``litellm/router_python_core.py``) is today's routing logic. The Rust core
+(litellm-rust/crates/router-core) implements the same operations. The core is picked once, when
+the router is built, from its ``RouterConfig``.
+"""
+
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Protocol, runtime_checkable
+
+from litellm.router_config import RouterConfig
+
+if TYPE_CHECKING:
+    from litellm.router_api import (
+        ApplyResult,
+        ChangeSet,
+        RouterHealth,
+        RouterRegistry,
+        RouterSettings,
+        RoutingObservers,
+    )
+    from litellm.router_python_core import PythonCore
+    from litellm.router_python_host import PythonHost
+    from litellm.types.router import CustomRoutingStrategyBase
+
+
+@runtime_checkable
+class RouterCore(Protocol):
+    async def call(self, op: str, request: Mapping[str, object]) -> object: ...
+
+    def registry(self) -> "RouterRegistry": ...
+
+    def settings(self) -> "RouterSettings": ...
+
+    def health(self) -> "RouterHealth": ...
+
+    async def select(
+        self,
+        model: str,
+        request_kwargs: dict[str, object] | None = None,  # mutable-ok: the selection cores take a dict
+    ) -> object: ...
+
+    def apply(self, changes: "ChangeSet") -> "ApplyResult": ...
+
+    def observe(self) -> "RoutingObservers": ...
+
+    def register_plugin(self, plugin: "CustomRoutingStrategyBase") -> None: ...
+
+    def retire(self) -> None: ...
+
+
+def make_core(config: RouterConfig, host: "PythonHost", core_class: "type[PythonCore] | None" = None) -> RouterCore:
+    from litellm.router_python_core import PythonCore
+
+    return (core_class or PythonCore)(**config.arguments, _host=host)  # pyright: ignore[reportArgumentType]  # the arguments were bound against PythonCore's own signature

litellm/router_deprecated.py

@@ -31,7 +31,7 @@ from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Generic, TypeVar, overload
 
-from litellm.router_api import SETTING_FIELDS, ChangeSet
+from litellm.router_api import ASSIGNABLE_ALIASES, SETTING_FIELDS, ChangeSet
 
 if TYPE_CHECKING:
     from typing import Any  # noqa: TID251  # only to repeat two old signatures that had Any, unchanged
@@ -65,7 +65,7 @@ _INTERNAL_PREFIX: Final = "litellm.router"
 
 
 # Router's own attribute hooks: a write or delete through an old name passes through them first.
-_ATTRIBUTE_HOOKS: Final = frozenset({"__setattr__", "__delattr__"})
+_ATTRIBUTE_HOOKS: Final = frozenset({"__getattribute__", "__setattr__", "__delattr__"})
 
 
 def _note(name: str, replacement: str) -> None:
@@ -88,9 +88,10 @@ def _note(name: str, replacement: str) -> None:
 def _as_router(obj: "DeprecatedRouterMembers") -> "Router":
     """The Router this mixin is part of (it is mixed into ``litellm.Router`` only)."""
     from litellm.router import Router
+    from litellm.router_python_core import PythonCore
 
-    if isinstance(obj, Router):
-        return obj
+    if isinstance(obj, (Router, PythonCore)):
+        return obj  # pyright: ignore[reportReturnType]  # a PythonCore carries every member the mixin reads
     raise TypeError(f"{type(obj).__name__} is not a litellm.Router; DeprecatedRouterMembers is mixed into Router only")
 
 
@@ -134,7 +135,7 @@ class _DeprecatedAttribute(Generic[_T]):
 
     def __set__(self, obj: "DeprecatedRouterMembers", value: _T) -> None:
         _note(self.name, self.replacement)
-        if self.name in SETTING_FIELDS:
+        if self.name in SETTING_FIELDS or self.name in ASSIGNABLE_ALIASES:
             _as_router(obj).apply(ChangeSet(assign={self.name: value}))
         else:
             setattr(obj, "_core_" + self.name, value)
@@ -169,6 +170,13 @@ _OVERRIDES_RUNNING: Final[contextvars.ContextVar[frozenset[tuple[int, str]]]] =
 )
 
 
+def _override_key(router: object, name: str) -> tuple[int, str]:
+    """The running-override key for ``router``: a Router and its core share one, so an old member
+    entered on the Router is seen by the override routing that runs on the core."""
+    core: Final = getattr(router, "__dict__", {}).get("_core")
+    return (id(router if core is None else core), name)
+
+
 def _enter_override(name: str, override: Callable[..., object]) -> Callable[..., object]:
     """
     A direct call of a subclass override (``router.<name>()``) marks it running, so its
@@ -184,7 +192,7 @@ def _enter_override(name: str, override: Callable[..., object]) -> Callable[...,
             *args: object,
             **kwargs: object,  # kwargs-ok: passes the override's arguments through unchanged
         ) -> object:
-            token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)})
+            token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)})
             try:
                 return await async_override(router, *args, **kwargs)
             finally:
@@ -198,7 +206,7 @@ def _enter_override(name: str, override: Callable[..., object]) -> Callable[...,
         *args: object,
         **kwargs: object,  # kwargs-ok: passes the override's arguments through unchanged
     ) -> object:
-        token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {(id(router), name)})
+        token: Final = _OVERRIDES_RUNNING.set(_OVERRIDES_RUNNING.get() | {_override_key(router, name)})
         try:
             return override(router, *args, **kwargs)
         finally:
@@ -236,7 +244,7 @@ def _route_override(cls: type, name: str, override: Callable[..., object]) -> Ca
             *args: object,
             **kwargs: object,  # kwargs-ok: passes the override's arguments through unchanged
         ) -> object:
-            key: Final = (id(router), name)
+            key: Final = _override_key(router, name)
             running: Final = _OVERRIDES_RUNNING.get()
             if key in running:
                 parent_async: Final[Callable[..., Awaitable[object]]] = getattr(super(cls, router), core)
@@ -255,7 +263,7 @@ def _route_override(cls: type, name: str, override: Callable[..., object]) -> Ca
         *args: object,
         **kwargs: object,  # kwargs-ok: passes the override's arguments through unchanged
     ) -> object:
-        key: Final = (id(router), name)
+        key: Final = _override_key(router, name)
         running: Final = _OVERRIDES_RUNNING.get()
         if key in running:
             return parent(router)(*args, **kwargs)
@@ -502,9 +510,9 @@ class DeprecatedRouterMembers:
     @staticmethod
     def deployment_usable_by_team(model: "Mapping[str, object] | Deployment", team_id: str | None) -> bool:
         _note("deployment_usable_by_team", "registry().includes(deployment, team_id=...)")
-        from litellm.router import Router
+        from litellm.router_python_core import PythonCore
 
-        return Router._core_deployment_usable_by_team(model, team_id)
+        return PythonCore._core_deployment_usable_by_team(model, team_id)
 
     _deployment_usable_by_team = deployment_usable_by_team
 

litellm/router_python_host.py

@@ -0,0 +1,135 @@
+"""
+``PythonHost``: what the router core reaches outside itself, served from the litellm package.
+
+The core never touches ``litellm.*`` directly. Inference calls (``invoke``), backoff sleeps,
+callback registration and the process-wide cache all go through the host, and the process settings
+come from ``dynamic()`` (``litellm/router_config.py``). A Rust core gets the same operations.
+"""
+
+import asyncio
+from collections.abc import Awaitable, Callable, Sequence
+from typing import (
+    TYPE_CHECKING,
+    Final,
+    Literal,
+    TypeAlias,
+    cast,  # noqa: TID251  # untyped litellm globals: the live config view and the callback lists
+)
+
+from typing_extensions import overload
+
+import litellm
+from litellm.router_config import RouterDynamicConfig, live_dynamic_config
+
+if TYPE_CHECKING:
+    from pydantic import BaseModel
+
+    from litellm.caching.caching import Cache
+    from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
+    from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
+    from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
+    from litellm.types.llms.openai import HttpxBinaryResponseContent, OpenAIFileObject
+    from litellm.types.rerank import RerankResponse
+    from litellm.types.utils import (
+        AdapterCompletionStreamWrapper,
+        EmbeddingResponse,
+        ImageResponse,
+        LiteLLMBatch,
+        ModelInfo,
+        ModelResponse,
+        TextCompletionResponse,
+        TranscriptionResponse,
+    )
+
+CallbackList: TypeAlias = Literal[
+    "success_callback", "_async_success_callback", "failure_callback", "_async_failure_callback", "input_callback"
+]
+
+Endpoint: TypeAlias = Callable[..., object]
+
+
+class PythonHost:
+    __slots__ = ()
+
+    # The endpoints whose results the router core reads, typed as litellm declares them; any other
+    # op gets the untyped Endpoint.
+    @overload
+    def invoke(self, op: Literal["acompletion"]) -> Callable[..., Awaitable["ModelResponse | CustomStreamWrapper"]]: ...
+    @overload
+    def invoke(
+        self, op: Literal["atext_completion"]
+    ) -> Callable[..., Awaitable["TextCompletionResponse | TextCompletionStreamWrapper"]]: ...
+    @overload
+    def invoke(
+        self, op: Literal["aadapter_completion"]
+    ) -> Callable[..., Awaitable["BaseModel | AdapterCompletionStreamWrapper | None"]]: ...
+    @overload
+    def invoke(self, op: Literal["aembedding"]) -> Callable[..., Awaitable["EmbeddingResponse"]]: ...
+    @overload
+    def invoke(self, op: Literal["aimage_generation"]) -> Callable[..., Awaitable["ImageResponse"]]: ...
+    @overload
+    def invoke(self, op: Literal["atranscription"]) -> Callable[..., Awaitable["TranscriptionResponse"]]: ...
+    @overload
+    def invoke(self, op: Literal["aspeech"]) -> Callable[..., Awaitable["HttpxBinaryResponseContent"]]: ...
+    @overload
+    def invoke(self, op: Literal["arerank"]) -> Callable[..., Awaitable["RerankResponse"]]: ...
+    @overload
+    def invoke(self, op: Literal["acreate_file"]) -> Callable[..., Awaitable["OpenAIFileObject"]]: ...
+    @overload
+    def invoke(self, op: Literal["acreate_batch"]) -> Callable[..., Awaitable["LiteLLMBatch"]]: ...
+    @overload
+    def invoke(self, op: Literal["aretrieve_batch"]) -> Callable[..., Awaitable["LiteLLMBatch"]]: ...
+    @overload
+    def invoke(self, op: Literal["acancel_batch"]) -> Callable[..., Awaitable["LiteLLMBatch"]]: ...
+    @overload
+    def invoke(self, op: Literal["alist_batches"]) -> Callable[..., Awaitable[object]]: ...
+    @overload
+    def invoke(self, op: Literal["get_llm_provider"]) -> Callable[..., "tuple[str, str, str | None, str | None]"]: ...
+    @overload
+    def invoke(self, op: Literal["get_model_info"]) -> Callable[..., "ModelInfo"]: ...
+    @overload
+    def invoke(self, op: Literal["get_supported_openai_params"]) -> Callable[..., "Sequence[str] | None"]: ...
+    @overload
+    def invoke(self, op: Literal["token_counter"]) -> Callable[..., int]: ...
+    @overload
+    def invoke(self, op: Literal["calculate_retry_after"]) -> Callable[..., float]: ...
+    @overload
+    def invoke(self, op: str) -> Endpoint: ...
+    def invoke(self, op: str) -> Endpoint:
+        """The litellm function named ``op`` (``acompletion``, ``get_model_info``, ...), looked up at call time."""
+        endpoint: Final[object] = getattr(litellm, op, None)
+        if endpoint is None:
+            raise AttributeError(f"litellm has no endpoint {op!r}")
+        if not callable(endpoint):
+            raise TypeError(f"litellm.{op} is not an endpoint")
+        return endpoint
+
+    def sleep(self, seconds: float) -> Awaitable[None]:
+        return asyncio.sleep(seconds)
+
+    def dynamic(self) -> RouterDynamicConfig:
+        """The process settings, read live on each access (``current_dynamic_config()`` takes a snapshot)."""
+        return cast(RouterDynamicConfig, live_dynamic_config)  # cast-ok: the live view reads the same fields
+
+    @property
+    def callbacks(self) -> "LoggingCallbackManager":
+        return litellm.logging_callback_manager
+
+    def callback_list(self, name: CallbackList) -> list[object]:  # mutable-ok: the process callback list itself
+        value: Final = getattr(litellm, name)
+        return cast(list[object], value)  # cast-ok: the process callback list is an untyped litellm global
+
+    def set_callback_list(self, name: CallbackList, value: list[object]) -> None:  # mutable-ok: assigned as given
+        setattr(litellm, name, value)
+
+    def global_cache(self) -> "Cache | None":
+        return litellm.cache
+
+    def set_global_cache(self, cache: "Cache | None") -> None:
+        litellm.cache = cache
+
+    def suppress_debug_info(self) -> None:
+        litellm.suppress_debug_info = True
+
+
+python_host: Final = PythonHost()

litellm/scheduler.py

@@ -1,15 +1,20 @@
 import asyncio
 import enum
 import heapq
+import logging as _logging
 import time
 from collections.abc import Awaitable, Callable, Sequence
 from typing import Final, TypeAlias
+from typing import Protocol as _Protocol
 
 from pydantic import TypeAdapter
 
 from litellm import print_verbose
 from litellm._internal_context import with_service_target
+from litellm._logging import verbose_logger as _verbose_logger
 from litellm.caching.caching import DualCache, RedisCache
+from litellm.caching.redis_cache import RedisCircuitBreakerOpenError as _RedisCircuitBreakerOpenError
+from litellm.caching.redis_cache import log_redis_failure as _log_redis_failure
 from litellm.constants import DEFAULT_IN_MEMORY_TTL, DEFAULT_POLLING_INTERVAL
 from litellm.exceptions import Timeout
 from litellm.types.llms.base import LiteLLMBaseModel
@@ -30,6 +35,26 @@ class FlowItem(LiteLLMBaseModel):
     model_name: str
 
 
+class _QueueCache(_Protocol):
+    async def async_get_cache(self, key: str, *, local_only: bool = False, raise_on_error: bool = False) -> object: ...
+
+    async def async_set_cache(self, key: str, value: object, *, local_only: bool = False) -> object: ...
+
+
+async def _read_queue(cache: _QueueCache, redis_cache: _QueueCache | None, key: str) -> object:
+    if redis_cache is None:
+        return await cache.async_get_cache(key=key, local_only=True)
+    from redis.exceptions import RedisError as _RedisError
+
+    try:
+        response: Final = await redis_cache.async_get_cache(key=key, raise_on_error=True)
+    except (_RedisError, _RedisCircuitBreakerOpenError) as error:
+        _log_redis_failure(_verbose_logger, _logging.ERROR, "Scheduler queue using local snapshot", error)
+        return await cache.async_get_cache(key=key, local_only=True)
+    await cache.async_set_cache(key=key, value=[] if response is None else response, local_only=True)
+    return response
+
+
 class Scheduler:
     cache: DualCache
 
@@ -153,7 +178,7 @@ class Scheduler:
         """
         if self.cache is not None:
             _cache_key: Final = f"{SchedulerCacheKeys.queue.value}:{model_name}"
-            response: Final = await self.cache.async_get_cache(key=_cache_key)
+            response: Final = await _read_queue(self.cache, self.cache.redis_cache, _cache_key)
             if not isinstance(response, list):
                 return []
             return _QUEUE_ENTRIES.validate_python(response)

litellm/types/utils.py

@@ -677,6 +677,12 @@ class CallTypes(str, Enum):
     retrieve_fine_tuning_job = "retrieve_fine_tuning_job"
     responses = "responses"
     aresponses = "aresponses"
+    get_responses = "get_responses"
+    aget_responses = "aget_responses"
+    delete_responses = "delete_responses"
+    adelete_responses = "adelete_responses"
+    cancel_responses = "cancel_responses"
+    acancel_responses = "acancel_responses"
     alist_input_items = "alist_input_items"
     llm_passthrough_route = "llm_passthrough_route"
     allm_passthrough_route = "allm_passthrough_route"

litellm/utils.py

@@ -1595,6 +1595,7 @@ def dispatch_success_logging(
     is_litellm_internal_call: bool,
 ) -> None:
     if is_litellm_internal_call:
+        logging_obj.capture_usage_before_logging(result, start_time, end_time)
         return
 
     _schedule_async_success_logging(
@@ -1633,9 +1634,14 @@ def _schedule_async_success_logging(
     if not logging_obj.claim_async_success_log():
         return
 
+    if logging_obj.defer_async_logging and not is_completion_with_fallbacks:
+        logging_obj.capture_usage_before_logging(result, start_time, end_time, freeze_only=True)
+
     def _enqueue_async_logging() -> None:
         from litellm.responses.streaming_iterator import spawn_logging_task
 
+        if not is_completion_with_fallbacks:
+            logging_obj.capture_usage_before_logging(result, start_time, end_time)
         if logging_obj.defer_async_logging:
             logging_obj.handle_sync_success_callbacks_for_async_calls(
                 result=result, start_time=start_time, end_time=end_time
@@ -2203,6 +2209,7 @@ def client(original_function):
 
             # LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's deprecated
             verbose_logger.info("Wrapper: Completed Call, calling success_handler")
+            logging_obj.capture_usage_before_logging(TypeAdapter(object).validate_python(result), start_time, end_time)
             if not is_internal_call.get():
                 # Copy the current context to propagate it to the background thread
                 # This is essential for OpenTelemetry span context propagation

model_prices_and_context_window.json

@@ -81372,5 +81372,77 @@
         "supports_response_schema": true,
         "supports_vision": true,
         "supports_xhigh_reasoning_effort": true
+    },
+    "gpt-6-astra-law": {
+        "cache_read_input_token_cost": 1e-06,
+        "input_cost_per_token": 1e-05,
+        "litellm_provider": "openai",
+        "max_input_tokens": 922000,
+        "max_output_tokens": 128000,
+        "mode": "chat",
+        "output_cost_per_token": 5e-05,
+        "reasoning_contexts": [
+            "auto",
+            "current_turn",
+            "all_turns"
+        ],
+        "reasoning_modes": [
+            "standard",
+            "pro"
+        ],
+        "regional_processing_uplift_multiplier_eu": 1.1,
+        "regional_processing_uplift_multiplier_us": 1.1,
+        "search_context_cost_per_query": {"search_context_size_high":0.01,"search_context_size_low":0.01,"search_context_size_medium":0.01},
+        "source": "https://developers.openai.com/api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/batch",
+            "/v1/responses"
+        ],
+        "supported_modalities": [
+            "text",
+            "image"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "supports_computer_use": true,
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_native_streaming": true,
+        "supports_none_reasoning_effort": false,
+        "supports_parallel_function_calling": true,
+        "supports_pdf_input": true,
+        "supports_prompt_cache_breakpoint": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_system_messages": true,
+        "supports_tool_choice": true,
+        "supports_tool_search": true,
+        "supports_vision": true,
+        "supports_web_search": true,
+        "supports_xhigh_reasoning_effort": true
+    },
+    "chatgpt/gpt-6-astra-law": {
+        "litellm_provider": "chatgpt",
+        "max_input_tokens": 922000,
+        "max_output_tokens": 128000,
+        "mode": "responses",
+        "source": "https://developers.openai.com/api/docs/pricing",
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/responses"
+        ],
+        "supports_function_calling": true,
+        "supports_max_reasoning_effort": true,
+        "supports_minimal_reasoning_effort": false,
+        "supports_none_reasoning_effort": false,
+        "supports_parallel_function_calling": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_xhigh_reasoning_effort": true
     }
 }

tests/code_coverage_tests/recursive_detector.py

@@ -3,6 +3,7 @@ import os
 
 IGNORE_FUNCTIONS = [
     "_json_cost",  # bounded at depth 32 and consumed under byte/node limits.
+    "detached",  # router_config: copies only the plain containers of the router's own constructor arguments, once per build or changeset
     "_format_type",
     "remove_additional_properties",
     "remove_strict_from_schema",

tests/code_coverage_tests/router_litellm_globals_check.py

@@ -0,0 +1,54 @@
+"""
+The router and its cores never read or write ``litellm.*``.
+
+Process settings reach the router through ``RouterDynamicConfig`` (litellm/router_config.py) and
+everything else outside it (inference calls, callbacks, the global cache) through ``PythonHost``
+(litellm/router_python_host.py). Those two modules are the only router code allowed to touch the
+``litellm`` package namespace. This check fails on ``import litellm`` (or ``import litellm.<x>``,
+which binds the same name) and on any ``litellm.<attr>`` access in the files below.
+
+Run: python tests/code_coverage_tests/router_litellm_globals_check.py
+"""
+
+import ast
+import os
+import sys
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
+CHECKED = (
+    "litellm/router.py",
+    "litellm/router_api.py",
+    "litellm/router_core.py",
+    "litellm/router_deprecated.py",
+    "litellm/router_python_core.py",
+)
+
+
+def violations(rel: str) -> list[str]:
+    tree = ast.parse(open(os.path.join(ROOT, rel)).read(), rel)
+    found = []
+    for node in ast.walk(tree):
+        if isinstance(node, ast.Import):
+            found.extend(
+                f"{rel}:{node.lineno}: import {alias.name}"
+                for alias in node.names
+                if alias.asname is None and (alias.name == "litellm" or alias.name.startswith("litellm."))
+            )
+        elif isinstance(node, ast.Attribute) and isinstance(node.value, ast.Name) and node.value.id == "litellm":
+            found.append(f"{rel}:{node.lineno}: litellm.{node.attr}")
+    return found
+
+
+def main() -> int:
+    found = [line for rel in CHECKED for line in violations(rel)]
+    for line in found:
+        print(line)
+    if found:
+        print(f"{len(found)} litellm.* uses in router modules; read settings from RouterDynamicConfig and use PythonHost")
+        return 1
+    print("router modules use no litellm.* globals")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/code_coverage_tests/router_surface_check.py

@@ -38,11 +38,15 @@ ALLOWLIST = os.path.join(os.path.dirname(__file__), "router_surface_allowlist.js
 ROUTER_FILES = {
     "litellm/router.py",
     "litellm/router_api.py",
+    "litellm/router_config.py",
+    "litellm/router_core.py",
+    "litellm/router_python_core.py",
+    "litellm/router_python_host.py",
     "litellm/router_deprecated.py",
     "litellm/router_host.py",
 }
-ROUTER_CLASS_FILES = ["litellm/router.py", "litellm/router_deprecated.py"]
-ROUTER_CLASS_NAMES = {"Router", "DeprecatedRouterMembers"}
+ROUTER_CLASS_FILES = ["litellm/router.py", "litellm/router_python_core.py", "litellm/router_deprecated.py"]
+ROUTER_CLASS_NAMES = {"Router", "PythonCore", "DeprecatedRouterMembers"}
 
 SCAN_ROOTS = ["litellm", "enterprise"]
 SKIP_DIRS = {

tests/integration/_support/anthropic_refusal.py

@@ -29,6 +29,7 @@ DOCS_STREAMING_REFUSAL_URL: Final = (
 )
 
 # Copied verbatim from DOCS_REFUSAL_URL and DOCS_STREAMING_REFUSAL_URL on 2026-10-12.
+DOCS_REFUSAL_EXPLANATION: Final = "This request was declined because it could enable cyber harm."
 DOCS_REFUSAL: Final[Event] = {
     "id": "msg_01XFUDYJgAACzvnptvVoYEL",
     "type": "message",
@@ -39,7 +40,7 @@ DOCS_REFUSAL: Final[Event] = {
     "stop_details": {
         "type": "refusal",
         "category": "cyber",
-        "explanation": "This request was declined because it could enable cyber harm.",
+        "explanation": DOCS_REFUSAL_EXPLANATION,
     },
     "usage": {"input_tokens": 412, "output_tokens": 0},
 }
@@ -50,7 +51,7 @@ DOCS_PARTIAL_REFUSAL: Final[Event] = {
     "stop_details": {
         "type": "refusal",
         "category": "cyber",
-        "explanation": "This request was declined because it could enable cyber harm.",
+        "explanation": DOCS_REFUSAL_EXPLANATION,
     },
 }
 

tests/integration/authorization/test_agent_access.py

@@ -1,7 +1,8 @@
+import itertools
 import json
 import uuid
 from collections.abc import Callable
-from typing import Final
+from typing import Final, Literal
 
 import pytest
 from integration._support.client import Gateway, Scenario
@@ -18,6 +19,22 @@ class _JsonRpcResult(BaseModel):
     result: dict[str, JsonValue]
 
 
+class _JsonRpcErrorBody(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    code: int
+    message: str
+    data: dict[str, JsonValue]
+
+
+class _JsonRpcError(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+
+    jsonrpc: Literal["2.0"]
+    id: str | int | None
+    error: _JsonRpcErrorBody
+
+
 _STREAM_EVENTS: Final = (
     {"kind": "status-update", "taskId": "t1", "contextId": "c1", "status": {"state": "working"}, "final": False},
     {
@@ -163,6 +180,14 @@ def test_agent_permissions_on_keys_teams_and_access_groups_gate_a2a_send_and_car
                 separators=(",", ":"),
             )
 
+        def jsonrpc_refusal(text: str, identity: str) -> bool:
+            error: Final = _JsonRpcError.model_validate_json(text)
+            return (error.id, error.error.code, error.error.message) == (
+                marker,
+                -32000,
+                f"Agent '{identity}' is not allowed for your key/team. Contact proxy admin for access.",
+            )
+
         spellings: Final = (
             ("send by id", lambda agent, name: f"/a2a/{agent}", send_payload, "message/send"),
             ("send by name", lambda agent, name: f"/a2a/{name}", send_payload, "message/send"),
@@ -224,13 +249,13 @@ def test_agent_permissions_on_keys_teams_and_access_groups_gate_a2a_send_and_car
 
         for label, key in callers.items():
             for spelling, status, text in attempt(key, denied, denied_name):
-                refused: Final = refusal(denied_name if "by name" in spelling else denied)
-                assert (status, text) == (403, refused), f"{label} {spelling}: {status} {text}"
+                identity: Final = denied_name if "by name" in spelling else denied
+                assert status == 403 and jsonrpc_refusal(text, identity), f"{label} {spelling}: {status} {text}"
             for spelling, payload in denied_tasks:
                 response: Final = gateway.client.post(
                     f"/a2a/{denied}", headers={"Authorization": f"Bearer {key}"}, json=payload
                 )
-                assert (response.status_code, response.text) == (403, refusal(denied)), (
+                assert response.status_code == 403 and jsonrpc_refusal(response.text, denied), (
                     f"{label} {spelling}: {response.status_code} {response.text}"
                 )
             assert card(key, denied) == (403, refusal(denied)), label
@@ -278,13 +303,72 @@ def test_agent_permissions_on_keys_teams_and_access_groups_gate_a2a_send_and_car
         emptied: Final = gateway.request("PUT", f"/v1/access_group/{group_id}", {"access_agent_ids": []})
         assert emptied.status_code == 200, emptied.text
         emptied_outcomes: Final = attempt(callers["access group"], allowed, allowed_name)
-        assert emptied_outcomes == tuple(
-            (spelling, 403, refusal(allowed_name if "by name" in spelling else allowed))
-            for spelling, _, _ in emptied_outcomes
+        assert all(
+            status == 403 and jsonrpc_refusal(text, allowed_name if "by name" in spelling else allowed)
+            for spelling, status, text in emptied_outcomes
         ), emptied_outcomes
         assert wire.drain() == (), emptied_outcomes
 
 
+def test_a2a_refusals_answer_as_jsonrpc_errors_that_echo_the_callers_request_id(gateway: Gateway) -> None:
+    marker: Final = "a2arefuse" + uuid.uuid4().hex[:12]
+
+    def upstream(request: Request) -> Reply:
+        return Reply(body=json.dumps(_peer_card(wire.url, marker)).encode())
+
+    with wire_server(upstream) as wire, gateway.scenario() as scenario:
+        agent: Final = _register_agent(gateway, scenario, marker, wire.url)
+        refused_callers: Final = (
+            (
+                "over budget",
+                scenario.key(object_permission={"agents": [agent]}, max_budget=0.0001, spend=1.0),
+                422,
+                "Budget has been exceeded!",
+                "budget_exceeded",
+            ),
+            (
+                "agent not granted",
+                scenario.key(object_permission={"agents": [marker + "-other"]}),
+                403,
+                f"Agent '{agent}' is not allowed for your key/team. Contact proxy admin for access.",
+                None,
+            ),
+            ("unknown key", "sk-" + uuid.uuid4().hex, 401, "Authentication Error", None),
+        )
+        message: Final = {
+            "kind": "message",
+            "role": "user",
+            "messageId": marker + "-in",
+            "parts": [{"kind": "text", "text": "ping"}],
+        }
+        requests: Final = tuple(
+            (path, method, request_id)
+            for path, (method, request_id) in itertools.product(
+                (f"/a2a/{agent}", f"/a2a/{agent}/message/send", f"/v1/a2a/{agent}/message/send"),
+                (("message/send", marker), ("message/stream", 7), ("SendMessage", "l10")),
+            )
+        )
+        for label, key, status, message_start, error_type in refused_callers:
+            for path, method, request_id in requests:
+                response = gateway.client.post(
+                    path,
+                    headers={"Authorization": f"Bearer {key}", "Accept": "application/json, text/event-stream"},
+                    json={"jsonrpc": "2.0", "id": request_id, "method": method, "params": {"message": message}},
+                )
+                context = f"{label} {path} {method}: {response.status_code} {response.text}"
+                assert response.status_code == status, context
+                error = _JsonRpcError.model_validate_json(response.text)
+                assert (error.id, error.error.code) == (request_id, -32000), context
+                assert error.error.message.startswith(message_start), context
+                assert error_type is None or error.error.data["type"] == error_type, context
+        unparseable: Final = gateway.client.post(
+            f"/a2a/{agent}", headers={"Authorization": "Bearer sk-" + uuid.uuid4().hex}, content=b"not json"
+        )
+        assert unparseable.status_code == 400, unparseable.text
+        assert _JsonRpcError.model_validate_json(unparseable.text).id is None, unparseable.text
+        assert wire.drain() == ()
+
+
 def test_legacy_agent_json_card_path_applies_agent_permissions_to_non_admin_keys(gateway: Gateway) -> None:
     pytest.skip(
         "BUG: GET /a2a/{agent_id}/.well-known/agent.json returns 401 'Only proxy admin can be used' for non-admin keys because agent_inference_routes lists only agent-card.json"

tests/integration/authorization/test_agent_least_privilege.py

@@ -221,8 +221,13 @@ def test_agent_access_default_deny_denies_ungranted_key_but_keeps_explicit_grant
         denied_sends: Final = tuple(
             _send(candidate, agent_id, key_a, f"denied-{index}-{uuid.uuid4().hex}") for index in range(8)
         )
-        _assert_denial(denied_sends[0], agent_id)
-        assert all(response.json() == denied_sends[0].json() for response in denied_sends), [
+        assert denied_sends[0].status_code == 403, denied_sends[0].text
+        assert denied_sends[0].json()["error"] == {
+            "code": -32000,
+            "message": f"Agent '{agent_id}' is not allowed for your key/team. Contact proxy admin for access.",
+            "data": denied_sends[0].json()["error"]["data"],
+        }, denied_sends[0].text
+        assert all(response.json()["error"] == denied_sends[0].json()["error"] for response in denied_sends), [
             response.text for response in denied_sends
         ]
         denied_completion: Final = _chat_completion(candidate, agent_name, key_a)

tests/integration/compatibility/test_a2a_wire_versions.py

@@ -348,6 +348,105 @@ _TRANSPORT_HEADERS: Final = frozenset(
 )
 
 
+def _a2a_auth_requests(
+    gateway: Gateway, method: str, agent_options: dict[str, JsonValue], client_headers: dict[str, str]
+) -> tuple[Request, ...]:
+    marker: Final = "a2akey" + uuid.uuid4().hex[:12]
+
+    def upstream(request: Request) -> Reply:
+        if request.method == "GET":
+            return Reply(body=json.dumps(_peer_card(wire.url, marker)).encode())
+        body: Final = json.loads(request.body)
+        if body["method"] == "message/stream":
+            frames: Final = tuple(
+                f"data: {json.dumps({'jsonrpc': '2.0', 'id': body['id'], 'result': event})}\n\n".encode()
+                for event in _STREAM_EVENTS
+            )
+            return Reply(content_type="text/event-stream", chunks=frames)
+        if body["method"] == "tasks/get":
+            return Reply(
+                body=json.dumps(
+                    {
+                        "jsonrpc": "2.0",
+                        "id": body["id"],
+                        "result": {"kind": "task", "id": "t1", "contextId": "c1", "status": {"state": "completed"}},
+                    }
+                ).encode()
+            )
+        return _message_reply(body, "key test")
+
+    with wire_server(upstream) as wire, gateway.scenario() as scenario:
+        identity: Final = _register_agent(
+            gateway,
+            scenario,
+            {"agent_name": marker, "agent_card_params": _peer_card(wire.url, marker, "1.0"), **agent_options},
+        )
+        params: Final = {"id": "t1"} if method == "tasks/get" else {"message": _MESSAGE_V10}
+        headers: Final = {
+            "Authorization": f"Bearer {gateway.key}",
+            "a2a-version": "1.0",
+            **{name.replace("{agent}", marker): value for name, value in client_headers.items()},
+        }
+        response: Final = gateway.client.post(
+            f"/a2a/{identity}/message/send",
+            headers=headers,
+            json={"jsonrpc": "2.0", "id": marker, "method": method, "params": params},
+        )
+        assert response.status_code == 200, response.text
+        assert "error" not in response.text, response.text
+        return wire.drain()
+
+
+@pytest.mark.parametrize(
+    ("method", "observed_method"),
+    (("SendMessage", "GET"), ("SendMessage", "POST"), ("SendStreamingMessage", "POST"), ("tasks/get", "POST")),
+    ids=("card", "send", "stream", "task"),
+)
+def test_regression_db_agent_api_key_is_bearer_on_each_a2a_hop(
+    gateway: Gateway, method: str, observed_method: str
+) -> None:
+    key: Final = "synthetic-backend-" + uuid.uuid4().hex
+    requests: Final = _a2a_auth_requests(gateway, method, {"litellm_params": {"api_key": key}}, {})
+    observed: Final = tuple(request for request in requests if request.method == observed_method)
+    assert len(observed) == 1, requests
+    assert observed[0].headers.get("authorization") == f"Bearer {key}", observed[0].headers
+
+
+@pytest.mark.parametrize("header_name", ("Authorization", "aUtHoRiZaTiOn"))
+def test_guard_static_authorization_overrides_api_key(gateway: Gateway, header_name: str) -> None:
+    requests: Final = _a2a_auth_requests(
+        gateway,
+        "SendMessage",
+        {"litellm_params": {"api_key": "backend-key"}, "static_headers": {header_name: "Bearer static-key"}},
+        {},
+    )
+    assert tuple(request.method for request in requests) == ("GET", "POST"), requests
+    assert tuple(request.headers.get("authorization") for request in requests) == ("Bearer static-key",) * 2
+
+
+@pytest.mark.parametrize("forwarding", ("configured", "convention"))
+def test_guard_forwarded_authorization_arrives_with_api_key(gateway: Gateway, forwarding: str) -> None:
+    options: Final = {
+        "litellm_params": {"api_key": "backend-key"},
+        **({"extra_headers": ["authorization"]} if forwarding == "configured" else {}),
+    }
+    client_headers: Final = (
+        {"x-litellm-api-key": gateway.key, "Authorization": "Bearer client-agent-key"}
+        if forwarding == "configured"
+        else {"x-a2a-{agent}-authorization": "Bearer client-agent-key"}
+    )
+    requests: Final = _a2a_auth_requests(gateway, "SendMessage", options, client_headers)
+    assert tuple(request.method for request in requests) == ("GET", "POST"), requests
+    assert tuple(request.headers.get("authorization") for request in requests) == ("Bearer client-agent-key",) * 2
+
+
+def test_guard_no_agent_key_or_proxy_caller_key_is_sent(gateway: Gateway) -> None:
+    requests: Final = _a2a_auth_requests(gateway, "SendMessage", {}, {})
+    assert tuple(request.method for request in requests) == ("GET", "POST"), requests
+    assert tuple(request.headers.get("authorization") for request in requests) == (None, None)
+    assert all(gateway.key not in value for request in requests for value in request.headers.values()), requests
+
+
 @pytest.mark.parametrize(
     ("version", "method", "message"),
     (("0.3", "message/stream", _MESSAGE_V03), ("1.0", "SendStreamingMessage", _MESSAGE_V10)),

tests/integration/cost_calculation/cases/misc.py

@@ -271,7 +271,7 @@ COHERE_RERANK_V4_TOTAL_TOKENS_FALLBACK: Final = CostTrackingTestCase(
             "meta": {"billed_units": {"total_tokens": 99}},
         },
     ),
-    expected=ExactExpected(spend=0.0, input_cost=0.0, output_cost=0.0, prompt_tokens=0, completion_tokens=0),
+    expected=ExactExpected(spend=0.0, input_cost=0.0, output_cost=0.0, prompt_tokens=99, completion_tokens=0),
 )
 
 
@@ -493,8 +493,8 @@ OPENROUTER_ANTHROPIC_CLAUDE_SONNET_5_PROVIDER_REPORTED_COST: Final = CostTrackin
     ),
     expected=ExactExpected(
         spend=0.0421,
-        input_cost=0.0,
-        output_cost=0.0421,
+        input_cost=0.0421 * (1840 * 3.2e-06) / (1840 * 3.2e-06 + 412 * 1.6e-05),
+        output_cost=0.0421 * (412 * 1.6e-05) / (1840 * 3.2e-06 + 412 * 1.6e-05),
         prompt_tokens=1840,
         completion_tokens=412,
         breakdown_persisted=False,

tests/integration/cost_calculation/cases/openai.py

@@ -7787,8 +7787,8 @@ GPT_5_6_UPSTREAM_COST_HEADER_BILLS_HEADER_VALUE: Final = CostTrackingTestCase(
     ),
     expected=ExactExpected(
         spend=0.000413,
-        input_cost=0.0,
-        output_cost=0.000413,
+        input_cost=0.000413 * (10 * 1.75e-06) / (10 * 1.75e-06 + 253 * 1.4e-05),
+        output_cost=0.000413 * (253 * 1.4e-05) / (10 * 1.75e-06 + 253 * 1.4e-05),
         prompt_tokens=10,
         completion_tokens=253,
         breakdown_persisted=False,
@@ -11885,7 +11885,7 @@ GPT_5_6_TERRA_MESSAGES_BRIDGED_STREAM: Final = GPT_5_6_TERRA_MESSAGES_BRIDGED_JS
 # replies alike. A deployment the operator priced keeps the operator's rates on both.
 _GATEWAY_REPORTED_USAGE: Final = {"prompt_tokens": 100, "completion_tokens": 20, "total_tokens": 120, "cost": 0.037}
 _GATEWAY_REPORTED_EXPECTED: Final = ExactExpected(
-    spend=0.037, input_cost=0.0, output_cost=0.037, prompt_tokens=100, completion_tokens=20
+    spend=0.037, input_cost=0.023125, output_cost=0.013875, prompt_tokens=100, completion_tokens=20
 )
 _GATEWAY_OPERATOR_PRICED: Final = Deployment(input_cost_per_token=2e-06, output_cost_per_token=6e-06)
 _GATEWAY_OPERATOR_PRICED_EXPECTED: Final = ExactExpected(

tests/integration/cost_calculation/cost_map.py

@@ -157,7 +157,7 @@ COST_MAP_ENTRIES: Final[dict[str, dict[str, JsonValue]]] = {
     "openrouter/typesafe/jev-1.13": {
         "litellm_provider": "openrouter",
         "mode": "evaluation",
-        "input_cost_per_token": 0.0,
+        "input_cost_per_token": 4.2e-08,
         "output_cost_per_token": 0.0,
     },
     "perplexity/pplx-decider-v1-27b": {

tests/integration/cost_calculation/test_messages_bridged_stream_margin_discount.py

@@ -0,0 +1,123 @@
+"""A streamed `/v1/messages` request served through the Responses bridge bills the same row as the plain request
+when a cost margin or a cost discount is configured. The streamed terminal event carries litellm's own bill in
+`usage.cost`, so the spend row must price the tokens the way the plain leg does instead of reading that bill back
+as the provider's charge, which books input 0 / output total and applies the margin or discount a second time.
+
+The stream_parity runner checks every leg against the suite's unadjusted expected row on the shared proxy, and a
+margin or discount set there would reprice every other case running beside it, so each pricing boots its own proxy."""
+
+from collections.abc import Iterator, Mapping
+from contextlib import ExitStack
+from dataclasses import dataclass
+from hashlib import sha256
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final
+
+import pytest
+import yaml
+from integration._support.client import Gateway, gateway_from_environment
+from integration._support.process import OwnedProxy, owned_proxy_process
+from integration._support.workers import cost_map_environment
+from integration.cost_calculation.cases.openai import (
+    GPT_6_LUNA_MESSAGES_BRIDGED_JSON,
+    GPT_6_LUNA_MESSAGES_BRIDGED_STREAM,
+)
+from integration.cost_calculation.conftest import CostRow, poll_rows, register_scenario_deployment
+from integration.cost_calculation.cost_tracking_case import CostTrackingTestCase, ExactExpected
+from pydantic import JsonValue, TypeAdapter
+
+pytestmark: Final = [pytest.mark.timeout(240), pytest.mark.shares_worker_state]
+
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_STOCK_CONFIG: Final = Path("tests/integration/proxy_config.yaml")
+_LEGS: Final[tuple[CostTrackingTestCase, ...]] = (GPT_6_LUNA_MESSAGES_BRIDGED_JSON, GPT_6_LUNA_MESSAGES_BRIDGED_STREAM)
+
+
+@dataclass(frozen=True, slots=True)
+class _Pricing:
+    name: str
+    litellm_settings: Mapping[str, JsonValue]
+    spend_multiplier: float
+
+
+PRICINGS: Final = (
+    _Pricing("margin", MappingProxyType({"cost_margin_config": {"global": 0.5}}), 1.5),
+    _Pricing("discount", MappingProxyType({"cost_discount_config": {"openai": 0.1}}), 0.9),
+)
+
+
+def _config(directory: Path, pricing: _Pricing) -> Path:
+    stock: Final = _JSON_OBJECT.validate_python(yaml.safe_load(_STOCK_CONFIG.read_text()))
+    settings: Final = _JSON_OBJECT.validate_python(stock["litellm_settings"])
+    path: Final = directory / f"{pricing.name}-config.yaml"
+    path.write_text(yaml.safe_dump({**stock, "litellm_settings": {**settings, **pricing.litellm_settings}}))
+    return path
+
+
+@pytest.fixture(scope="module")
+def proxies(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Mapping[str, OwnedProxy]]:
+    with gateway_from_environment() as gateway, ExitStack() as stack:
+        overrides: Final = cost_map_environment(gateway.upstream_url)
+
+        def boot(pricing: _Pricing) -> OwnedProxy:
+            directory: Final = tmp_path_factory.mktemp(f"messages-bridged-stream-{pricing.name}-rig")
+            return stack.enter_context(
+                owned_proxy_process(
+                    gateway,
+                    directory,
+                    overrides,
+                    config=_config(directory, pricing),
+                    remove_environment=("LITELLM_LOCAL_MODEL_COST_MAP",),
+                )
+            )
+
+        yield MappingProxyType({pricing.name: boot(pricing) for pricing in PRICINGS})
+
+
+def _bill_legs(gateway: Gateway) -> Mapping[str, CostRow]:
+    with gateway.scenario() as scenario:
+        key: Final = scenario.key()
+        deployments: Final = {
+            leg.name: register_scenario_deployment(scenario, leg, sha256(leg.name.encode()).hexdigest()[:12], key)
+            for leg in _LEGS
+        }
+        responses: Final = {
+            leg.name: gateway.request(
+                "POST", leg.endpoint, {**leg.request, "model": deployments[leg.name].model_name}, key=key
+            )
+            for leg in _LEGS
+        }
+        for leg in _LEGS:
+            assert responses[leg.name].status_code == 200, (leg.name, responses[leg.name].text[:400])
+        rows: Final = poll_rows(key, len(_LEGS))
+        return MappingProxyType(
+            {leg.name: next(row for row in rows if row.model_id == deployments[leg.name].identity) for leg in _LEGS}
+        )
+
+
+def _priced(row: CostRow) -> tuple[float | None, float | None, float | None, int | None, int | None]:
+    breakdown: Final = row.breakdown
+    assert breakdown is not None, row
+    return (row.spend, breakdown.input_cost, breakdown.output_cost, row.prompt_tokens, row.completion_tokens)
+
+
+@pytest.mark.parametrize("pricing", PRICINGS, ids=[pricing.name for pricing in PRICINGS])
+def test_streamed_messages_bridged_row_matches_the_plain_row_under_a_margin_or_discount(
+    proxies: Mapping[str, OwnedProxy], pricing: _Pricing
+) -> None:
+    billed: Final = _bill_legs(proxies[pricing.name].gateway)
+    plain: Final = billed[GPT_6_LUNA_MESSAGES_BRIDGED_JSON.name]
+    streamed: Final = billed[GPT_6_LUNA_MESSAGES_BRIDGED_STREAM.name]
+    expected: Final = GPT_6_LUNA_MESSAGES_BRIDGED_JSON.expected
+    assert isinstance(expected, ExactExpected), expected
+    assert _priced(plain) == pytest.approx(
+        (
+            expected.spend * pricing.spend_multiplier,
+            expected.input_cost,
+            expected.output_cost,
+            expected.prompt_tokens,
+            expected.completion_tokens,
+        )
+    ), plain
+    assert _priced(streamed) == pytest.approx(_priced(plain)), (plain, streamed)

tests/integration/database/test_accounting_ledger.py

@@ -540,7 +540,12 @@ async def test_usage_batch_is_atomic_and_replay_preserves_receipts(
     writer: _Writer, ledger_database: str, failure: str
 ) -> None:
     client, store = writer
-    events: Final = (_usage("caller", inbound=True), _usage("attempt"))
+    events: Final = (
+        _usage("caller", inbound=True),
+        _usage("attempt").model_copy(
+            update={"metrics": (Metric(name=MetricName.PROMPT_TOKENS, value=Decimal("120000")),)}
+        ),
+    )
     writes: Final = tuple(UsageWrite(event, _envelope(event), _TARGETS) for event in events)
 
     @asynccontextmanager

tests/integration/observability/test_arize_otel_v2_openinference_modes.py

@@ -24,6 +24,7 @@ from _openinference_support import (
     _rig,
 )
 from integration._support.client import Gateway
+from integration._support.process import graceful_stop_seconds
 from integration._support.wire import Reply, Request
 
 _DEFAULT_METADATA: Final = {
@@ -411,7 +412,7 @@ def test_arize_otel_v2_c9_upstream_failures_are_recorded(failure_status: int, ga
         assert not any(".tool_calls." in key for key in attributes), attributes
 
 
-@pytest.mark.timeout(180)
+@pytest.mark.timeout(4 * graceful_stop_seconds() + 120)
 def test_arize_otel_v2_c10_attribute_limit_keeps_tool_prefix(gateway: Gateway, tmp_path: Path) -> None:
     marker: Final = "c10-" + uuid.uuid4().hex
     calls: Final = _response_tool_calls(marker, ("Paris", "Berlin", "Rome", "Tokyo", "Oslo", "Lima", "Accra", "Delhi"))

tests/integration/observability/test_logging_only_scope_chaos.py

@@ -56,7 +56,7 @@ pytestmark: Final = pytest.mark.shares_worker_state
 _EDGE_DELAY_SECONDS: Final = 5
 
 
-@pytest.mark.timeout(180)
+@pytest.mark.timeout(4 * graceful_stop_seconds() + 120)
 def test_K1_policy_edge_restart_mid_burst_keeps_output_observation_fail_open(gateway: Gateway, tmp_path: Path) -> None:
     identity: Final = f"logging-scope-k1-{uuid.uuid4().hex}"
     marker: Final = uuid.uuid4().hex
@@ -215,7 +215,7 @@ def test_K1_policy_edge_restart_mid_burst_keeps_output_observation_fail_open(gat
                 ) == tuple((identity, "logging_only", expected_status) for _ in expected_directions), (index, entries)
 
 
-@pytest.mark.timeout(180)
+@pytest.mark.timeout(4 * graceful_stop_seconds() + 120)
 def test_K2_policy_edge_delay_does_not_delay_concurrent_callers(gateway: Gateway, tmp_path: Path) -> None:
     identity: Final = f"logging-scope-k2-{uuid.uuid4().hex}"
     marker: Final = uuid.uuid4().hex

tests/integration/observability/test_otel_team_capture_message_content.py

@@ -303,6 +303,23 @@ def _with_content(url: str, since: int, sent: Sent) -> tuple[Span, ...]:
     )
 
 
+def _is_root(span: Span) -> bool:
+    return span["kind"] == SERVER and span["name"].startswith("POST")
+
+
+def _with_root(url: str, since: int, sent: Sent) -> tuple[Span, ...]:
+    """The copy of ``sent`` at ``url`` once its model span, the marker and its server span arrived."""
+    return eventually(
+        lambda: _of_request(url, since, sent),
+        lambda spans: (
+            bool(_model_spans(spans))
+            and any(_carries(span, sent.marker) for span in spans)
+            and any(_is_root(span) for span in spans)
+        ),
+        seconds=40,
+    )
+
+
 def _arrived(
     url: str, since: int, sent: Sent, expected: frozenset[str], credential: Credential | None = None
 ) -> tuple[Span, ...]:
@@ -626,11 +643,7 @@ def test_stalled_tenant_during_a_mixed_burst_keeps_serving_and_each_teams_policy
         configure_sink(rig.sinks.tenant, paused=False)
     assert [result.status for result in sent] == [200] * 30, [result.text for result in sent if result.status != 200]
     for (key, _, _), result in zip(plan, sent, strict=True):
-        roots = [
-            span
-            for span in _with_content(rig.sinks.operator, cursors.operator, result)
-            if span["kind"] == SERVER and span["name"].startswith("POST ")
-        ]
+        roots = [span for span in _with_root(rig.sinks.operator, cursors.operator, result) if _is_root(span)]
         assert len(roots) == 1, roots
         if key == restricted:
             _assert_redacted_twin(rig, result, cursors, rig.sinks.tenant, cursors.tenant)

tests/integration/observability/test_s3_v2_partition_granularity.py

@@ -30,7 +30,7 @@ from _s3_v2_support import (
 from integration._support.client import Gateway, JsonValue, Scenario, eventually, object_value
 from integration._support.database import read_rows, scratch_database
 from integration._support.database_relay import database_relay
-from integration._support.process import OwnedProxy, owned_proxy_process
+from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
 from integration._support.wire import Reply, Request, wire_server
 
 pytestmark: Final = pytest.mark.shares_worker_state
@@ -1160,7 +1160,7 @@ def test_s3_v2_hour_worker_kill_mid_burst_keeps_the_other_worker_logging(gateway
     assert _outside_layout(objects, "hour") == ()
 
 
-@pytest.mark.timeout(180)
+@pytest.mark.timeout(4 * graceful_stop_seconds() + 120)
 def test_s3_v2_hour_proxy_restart_mid_burst_keeps_the_layout_without_duplicates(
     gateway: Gateway, tmp_path: Path
 ) -> None:

tests/integration/providers/test_anthropic_refusal_stop_details_wire.py

@@ -18,6 +18,7 @@ from integration._support.anthropic_refusal import (
     DECLARED_PARTIAL_JSON,
     DOCS_PARTIAL_TEXT,
     DOCS_REFUSAL,
+    DOCS_REFUSAL_EXPLANATION,
     DOCS_STOP_DETAILS,
     ENVELOPED_PARTIAL_JSON,
     FIXTURES,
@@ -651,8 +652,7 @@ def test_family_messages_stream_refusal_keeps_frames_and_logs_the_stop_details(r
         model: Final = _deployment(scenario, provider, rig)
         key: Final = scenario.key()
         events: Final = _messages_stream(rig, key, model, refusal_prompt("refusal", _marker()))
-        native: Final = {"stop_reason": "refusal"} if provider == "snowflake" else REFUSAL_FIELDS
-        assert _native_deltas(events) == (native,), events
+        assert _native_deltas(events) == (REFUSAL_FIELDS,), events
         assert _relayed_in(events) == (), events
         assert _paths(rig.wire) == (_STREAM_TARGETS[provider][1],)
         logged: Final = _logged_response(rig, key)
@@ -783,10 +783,10 @@ def test_responses_cache_hit_logs_the_same_stop_details_as_the_miss(rig: Rig) ->
         served_from_cache: Final = tuple(event.get("cache_hit") is True for event in logged)
         assert (len(_paths(rig.wire)), served_from_cache.count(True)) == (1, 1), (served_from_cache, logged)
         assert relayed(first) == ((_RESPONSE_FIELDS, REFUSAL_FIELDS),), first
-        assert tuple(relayed(event["response"]) for event in logged) == (((_CHAT_FIELDS, REFUSAL_FIELDS),),) * 2, (
-            served_from_cache,
-            logged,
-        )
+        assert tuple(relayed(event["response"]) for event in logged) == (
+            ((_CHAT_FIELDS, REFUSAL_FIELDS),),
+            relayed(first),
+        ), (served_from_cache, logged)
 
 
 def test_passthrough_is_not_served_from_the_response_cache(rig: Rig) -> None:
@@ -1310,6 +1310,7 @@ def test_chat_refusal_body_serializes_the_stop_details_beside_the_existing_messa
             "message": {
                 "content": content,
                 "role": "assistant",
+                "refusal": DOCS_REFUSAL_EXPLANATION,
                 "provider_specific_fields": {**_ANTHROPIC_MESSAGE_FIELDS, **REFUSAL_FIELDS},
             },
         }

tests/integration/providers/test_cache_control_tool_call_marks_wire.py

@@ -354,7 +354,7 @@ _WEB_RESULTS: Final[dict[str, JsonValue]] = {
         ),
         pytest.param(
             {},
-            [ASK_LABEL, tool_use_label(CITIES[0]), tool_use_label(CITIES[1]), "assistant:tool_use:srvtoolu_web"],
+            [SYSTEM_LABEL, ASK_LABEL, tool_use_label(CITIES[0]), tool_use_label(CITIES[1])],
             id="server-tool-without-result",
         ),
     ),
@@ -413,16 +413,14 @@ _GEMINI_CACHE_WRITE: Final = [
 
 
 @pytest.mark.parametrize(
-    ("calls", "expected"),
+    "calls",
     (
-        pytest.param(
-            marked_calls(), [("POST", "/models/gemini-3.8-flash:generateContent")], id="tool-calls-fill-the-cap"
-        ),
-        pytest.param([tool_call(city) for city in CITIES], _GEMINI_CACHE_WRITE, id="unmarked-tool-calls"),
+        pytest.param(marked_calls(), id="tool-calls-fill-the-cap"),
+        pytest.param([tool_call(city) for city in CITIES], id="unmarked-tool-calls"),
     ),
 )
-def test_gemini_context_cache_follows_the_cap_census(
-    gateway: Gateway, calls: list[JsonValue], expected: list[tuple[str, str]]
+def test_gemini_context_cache_takes_the_system_message_with_the_marked_ask(
+    gateway: Gateway, calls: list[JsonValue]
 ) -> None:
     marker: Final = new_marker()
     long_system: Final[JsonValue] = {"role": "system", "content": "lorem " * (2 * get_prompt_cache_min_tokens(_GEMINI))}
@@ -436,8 +434,18 @@ def test_gemini_context_cache_follows_the_cap_census(
         )
         status, _, text = post_chat(gateway, chat_body(model, messages))
         assert status == 200, text
-        targets: Final = [(request.method, request.target.split("?")[0]) for request in wire.drain()]
-    assert targets == expected
+        received: Final = wire.drain()
+    targets: Final = [(request.method, request.target.split("?")[0]) for request in received]
+    cache_writes: Final = [
+        request.body for request in received if request.method == "POST" and "cachedContents" in request.target
+    ]
+    generate: Final = _JSON_OBJECT.validate_json(received[-1].body)
+    assert (
+        targets,
+        [b"lorem lorem" in body for body in cache_writes],
+        generate.get("cachedContent"),
+        "system_instruction" in generate,
+    ) == (_GEMINI_CACHE_WRITE, [True], "cachedContents/census", False), (cache_writes, generate)
 
 
 def _openai_marks(request: Request) -> list[str]:
@@ -747,11 +755,11 @@ def test_request_level_points_skip_injection_when_client_marks_fill_the_cap(gate
 _OTHER_CALL_RESULT: Final[dict[str, JsonValue]] = {
     "web_search_results": [{"type": "web_search_tool_result", "tool_use_id": "srvtoolu_other", "content": []}]
 }
-_SERVER_MARK_COUNTED: Final = [
+_UNRUN_SERVER_MARK_UNCOUNTED: Final = [
+    SYSTEM_LABEL,
     ASK_LABEL,
     tool_use_label(CITIES[0]),
     tool_use_label(CITIES[1]),
-    "assistant:tool_use:srvtoolu_web",
 ]
 
 
@@ -767,7 +775,7 @@ _SERVER_MARK_COUNTED: Final = [
         pytest.param(_OTHER_CALL_RESULT, id="result-for-another-call"),
     ),
 )
-def test_malformed_provider_specific_fields_count_the_server_tool_call_mark(
+def test_malformed_provider_specific_fields_leave_the_unrun_server_tool_call_mark_uncounted(
     gateway: Gateway, fields: JsonValue
 ) -> None:
     marker: Final = new_marker()
@@ -779,7 +787,7 @@ def test_malformed_provider_specific_fields_count_the_server_tool_call_mark(
         )
         assert status == 200, text
         labels: Final = anthropic_labels(_only_request(wire))
-    assert labels == _SERVER_MARK_COUNTED
+    assert labels == _UNRUN_SERVER_MARK_UNCOUNTED
 
 
 @pytest.mark.parametrize(

tests/integration/providers/test_watsonx_passthrough_router_credentials_wire.py

@@ -0,0 +1,81 @@
+import json
+import uuid
+from pathlib import Path
+from typing import Final
+
+import yaml
+from integration._support.client import Gateway, JsonValue, object_value
+from integration._support.process import owned_proxy
+from integration._support.wire import Reply, Request, wire_server
+
+_DEPLOYMENT_KEY: Final = "synthetic-watsonx-deployment-key"
+_IAM_TOKEN: Final = "synthetic-watsonx-iam-token"
+_WATSONX_ENV: Final = (
+    "WATSONX_API_KEY",
+    "WATSONX_APIKEY",
+    "WX_API_KEY",
+    "WATSONX_API_BASE",
+    "WATSONX_URL",
+    "WX_URL",
+    "WML_URL",
+    "WATSONX_TOKEN",
+    "WATSONX_ZENAPIKEY",
+)
+
+
+def _config(directory: Path, api_base: str) -> Path:
+    stock: Final = object_value(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
+    config: Final[dict[str, JsonValue]] = {
+        **stock,
+        "model_list": [
+            {
+                "model_name": "granite",
+                "litellm_params": {
+                    "model": "watsonx/ibm/granite-3-8b-instruct",
+                    "api_key": _DEPLOYMENT_KEY,
+                    "api_base": api_base,
+                    "project_id": "synthetic-project",
+                    "use_in_pass_through": True,
+                },
+            }
+        ],
+    }
+    path: Final = directory / "watsonx-passthrough.yaml"
+    path.write_text(yaml.safe_dump(config))
+    return path
+
+
+def test_watsonx_passthrough_uses_the_flagged_deployment_key_and_base(gateway: Gateway, tmp_path: Path) -> None:
+    marker: Final = "watsonx-" + uuid.uuid4().hex
+
+    def respond(request: Request) -> Reply:
+        if request.method == "POST" and request.target == "/identity/token":
+            assert f"apikey={_DEPLOYMENT_KEY}".encode() in request.body, request.body
+            return Reply(body=json.dumps({"access_token": _IAM_TOKEN, "expires_in": 3600}).encode())
+        if request.method == "POST" and request.target.startswith("/ml/v1/text/generation?"):
+            assert request.headers["authorization"] == f"Bearer {_IAM_TOKEN}", request.headers
+            assert marker.encode() in request.body, request.body[:400]
+            return Reply(
+                body=json.dumps(
+                    {"model_id": "ibm/granite-3-8b-instruct", "results": [{"generated_text": marker}]}
+                ).encode()
+            )
+        return Reply(status=404)
+
+    with wire_server(respond) as wire:
+        with owned_proxy(
+            gateway,
+            tmp_path,
+            {"WATSONX_IAM_URL": f"{wire.url}/identity/token"},
+            config=_config(tmp_path, wire.url),
+            remove_environment=_WATSONX_ENV,
+        ) as candidate:
+            response: Final = candidate.request(
+                "POST",
+                "/watsonx/ml/v1/text/generation",
+                {"model_id": "ibm/granite-3-8b-instruct", "input": marker, "project_id": "synthetic-project"},
+            )
+            assert response.status_code == 200, response.text
+            assert response.json()["results"][0]["generated_text"] == marker
+        forwarded: Final = tuple((request.method, request.target.split("?")[0]) for request in wire.drain())
+        assert forwarded == (("POST", "/identity/token"), ("POST", "/ml/v1/text/generation")), forwarded

tests/integration/streaming/test_decision_model_buffering.py

@@ -229,11 +229,17 @@ def _frames(identity: str, endpoint: str) -> tuple[str, ...]:
                         'object': 'chat.completion.chunk',
                         'created': 1,
                         'model': 'scripted-chat',
-                        'choices': [{'index': 0, 'delta': {'content': text}, 'finish_reason': None}],
+                        'choices': [
+                            {
+                                'index': 0,
+                                'delta': {'content': text},
+                                'finish_reason': 'stop' if index == len(pieces) - 1 else None,
+                            }
+                        ],
                     }
                 )
             }"
-            for text in pieces
+            for index, text in enumerate(pieces)
         ),
         "data: [DONE]",
     )

tests/integration/streaming/test_stream_clean_eof_without_finish_reason.py

@@ -0,0 +1,564 @@
+import asyncio
+import itertools
+import json
+import uuid
+from collections import Counter
+from collections.abc import Callable, Iterator, Mapping
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import ExitStack
+from pathlib import Path
+from typing import Final, Literal
+
+import anthropic
+import httpx
+import openai
+import pytest
+from integration._support.anthropic_sse import CONTENT_BLOCK_START, CONTENT_TAIL, message_start, text_delta
+from integration._support.client import Gateway, Scenario, eventually, string_value
+from integration._support.database import read_rows, scratch_database
+from integration._support.database_relay import database_relay, plaintext_database_url
+from integration._support.openai_wire import answering_model_discovery, posted_targets
+from integration._support.process import owned_proxy, owned_proxy_process
+from integration._support.responses_stream import data_frames, spend_row, sse_events
+from integration._support.wire import Reply, Request, Wire, wire_server
+from integration.providers._count_tokens_system_lift import (
+    anthropic_client,
+    async_anthropic_client,
+    async_openai_client,
+    openai_client,
+)
+from pydantic import JsonValue
+
+Ending = Literal["ending-complete", "ending-clean-eof", "ending-done-without-finish", "ending-done-only"]
+Family = Literal["openai", "anthropic", "hosted_vllm", "gemini", "langgraph", "agentcore"]
+
+_ENDINGS: Final[tuple[Ending, ...]] = (
+    "ending-complete",
+    "ending-clean-eof",
+    "ending-done-without-finish",
+    "ending-done-only",
+)
+_PARTIAL: Final = ('{"findings":[{"title":', '"unfinished')
+_BACKEND: Final = "gpt-5.6"
+_CLAUDE: Final = "claude-sonnet-5-5"
+_GEMINI: Final = "gemini-3.8-flash"
+_AGENT_RUNTIME: Final = "arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/integration-agent"
+_PROVIDER_KEY: Final = "synthetic-provider-key"
+_CALL_ID_HEADER: Final = "x-litellm-call-id"
+_USAGE: Final = {"prompt_tokens": 9, "completion_tokens": 7, "total_tokens": 16}
+_TARGETS: Final[Mapping[Family, str]] = {
+    "openai": "/chat/completions",
+    "hosted_vllm": "/chat/completions",
+    "anthropic": "/v1/messages",
+    "gemini": f"/models/{_GEMINI}:streamGenerateContent",
+    "langgraph": "/runs/stream",
+    "agentcore": "/runtimes/",
+}
+_DEPLOYMENTS: Final[Mapping[Family, str]] = {
+    "openai": f"openai/{_BACKEND}",
+    "hosted_vllm": f"hosted_vllm/{_BACKEND}",
+    "anthropic": f"anthropic/{_CLAUDE}",
+    "gemini": f"gemini/{_GEMINI}",
+    "langgraph": "langgraph/agent",
+    "agentcore": f"bedrock/agentcore/{_AGENT_RUNTIME}",
+}
+
+
+def _data(frame: JsonValue) -> bytes:
+    return b"data: " + json.dumps(frame).encode() + b"\n\n"
+
+
+def _chat_frames(identity: str, ending: Ending) -> tuple[bytes, ...]:
+    chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": _BACKEND}
+    content: Final = (
+        _data({**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": _PARTIAL[0]}}]}),
+        _data({**chunk, "choices": [{"index": 0, "delta": {"content": _PARTIAL[1]}, "finish_reason": None}]}),
+    )
+    match ending:
+        case "ending-complete":
+            finish: Final = _data({**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]})
+            usage: Final = _data({**chunk, "choices": [], "usage": _USAGE})
+            return (*content, finish + usage + b"data: [DONE]\n\n")
+        case "ending-clean-eof":
+            return content
+        case "ending-done-without-finish":
+            return (*content, b"data: [DONE]\n\n")
+        case "ending-done-only":
+            return (b"data: [DONE]\n\n",)
+
+
+def _anthropic_frames(identity: str, ending: Ending) -> tuple[bytes, ...]:
+    content: Final = (message_start(identity, _CLAUDE), CONTENT_BLOCK_START, *map(text_delta, _PARTIAL))
+    return (*content, CONTENT_TAIL) if ending == "ending-complete" else content
+
+
+def _gemini_frames(ending: Ending) -> tuple[bytes, ...]:
+    """Gemini puts finishReason on the last text chunk and can follow it with a usage-only chunk."""
+    finished: Final = ending == "ending-complete"
+    content: Final = tuple(
+        _data(
+            {
+                "candidates": [
+                    {
+                        "content": {"parts": [{"text": part}], "role": "model"},
+                        "index": 0,
+                        **({"finishReason": "STOP"} if finished and part == _PARTIAL[-1] else {}),
+                    }
+                ]
+            }
+        )
+        for part in _PARTIAL
+    )
+    usage: Final = _data({"usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 7, "totalTokenCount": 16}})
+    return (*content, usage) if finished else content
+
+
+def _event(name: str, frame: JsonValue) -> bytes:
+    return f"event: {name}\n".encode() + _data(frame)
+
+
+def _langgraph_frames() -> tuple[bytes, ...]:
+    """A LangGraph server's run stream in its default mode, its run metadata first and then the state after each step"""
+    question: Final = {"type": "human", "content": "Answer in two steps."}
+    answers: Final = tuple({"type": "ai", "content": part} for part in _PARTIAL)
+    return (
+        _event("metadata", {"run_id": "run-1", "attempt": 1}),
+        *(_event("values", {"messages": [question, *answers[:step]]}) for step in range(len(answers) + 1)),
+    )
+
+
+def _agentcore_frames(ending: Ending) -> tuple[bytes, ...]:
+    """A runtime that forwards Bedrock Converse events without the metadata event, so its turn ends on messageStop"""
+    start: Final = _data({"event": {"messageStart": {"role": "assistant"}}})
+    content: Final = tuple(
+        _data({"event": {"contentBlockDelta": {"delta": {"text": part}, "contentBlockIndex": 0}}}) for part in _PARTIAL
+    )
+    end: Final = (
+        _data({"event": {"contentBlockStop": {"contentBlockIndex": 0}}}),
+        _data({"event": {"messageStop": {"stopReason": "end_turn"}}}),
+    )
+    return (start, *content, *end) if ending == "ending-complete" else (start, *content)
+
+
+def _frames(family: Family, identity: str, ending: Ending) -> tuple[bytes, ...]:
+    match family:
+        case "agentcore":
+            return _agentcore_frames(ending)
+        case "anthropic":
+            return _anthropic_frames(identity, ending)
+        case "gemini":
+            return _gemini_frames(ending)
+        case "langgraph":
+            return _langgraph_frames()
+        case "openai" | "hosted_vllm":
+            return _chat_frames(identity, ending)
+
+
+def _ending(request: Request) -> Ending:
+    endings: Final = [ending for ending in _ENDINGS if ending.encode() in request.body]
+    assert len(endings) == 1, request.body
+    return endings[0]
+
+
+def _upstream(family: Family, ending: Ending | None = None) -> Callable[[Request], Reply]:
+    def respond(request: Request) -> Reply:
+        assert request.method == "POST" and _TARGETS[family] in request.target, request.target
+        frames: Final = _frames(family, f"upstream-{uuid.uuid4().hex}", ending or _ending(request))
+        return Reply(content_type="text/event-stream", chunks=frames)
+
+    return answering_model_discovery(respond)
+
+
+def _deploy(scenario: Scenario, family: Family, wire: Wire) -> str:
+    if family == "agentcore":
+        return scenario.model(
+            model=_DEPLOYMENTS[family],
+            aws_bedrock_runtime_endpoint=wire.url,
+            aws_access_key_id="AKIAINTEGRATIONAGENT",
+            aws_secret_access_key="synthetic-agentcore-secret",
+            aws_region_name="us-west-2",
+        )
+    return scenario.model(model=_DEPLOYMENTS[family], api_base=wire.url, api_key=_PROVIDER_KEY)
+
+
+def _prompt(ending: Ending) -> str:
+    return f"Return JSON {ending} {uuid.uuid4().hex}"
+
+
+def _post(gateway: Gateway, path: str, body: Mapping[str, JsonValue]) -> tuple[str, str]:
+    response: Final = gateway.request("POST", path, {**body, "stream": True, "num_retries": 0})
+    assert response.status_code == 200, response.text
+    return response.text, response.headers[_CALL_ID_HEADER]
+
+
+def _chat(gateway: Gateway, model: str, ending: Ending) -> tuple[str, str]:
+    return _post(
+        gateway,
+        "/v1/chat/completions",
+        {
+            "model": model,
+            "messages": [{"role": "user", "content": _prompt(ending)}],
+            "stream_options": {"include_usage": True},
+        },
+    )
+
+
+def _choices(text: str) -> Iterator[Mapping[str, JsonValue]]:
+    for frame in data_frames(text):
+        yield from (choice for choice in frame.get("choices") or [] if isinstance(choice, dict))
+
+
+def _finish_reasons(text: str) -> tuple[str, ...]:
+    return tuple(string_value(choice["finish_reason"]) for choice in _choices(text) if choice.get("finish_reason"))
+
+
+def _chat_text(text: str) -> str:
+    deltas: Final = (choice.get("delta") for choice in _choices(text))
+    return "".join(string_value(delta.get("content") or "") for delta in deltas if isinstance(delta, dict))
+
+
+def _chat_errors(text: str) -> tuple[Mapping[str, JsonValue], ...]:
+    return tuple(frame for frame in data_frames(text) if "error" in frame)
+
+
+def _event_types(text: str) -> tuple[str, ...]:
+    return tuple(event for event, _ in sse_events(text))
+
+
+@pytest.mark.parametrize("family", ["openai", "anthropic", "agentcore"])
+def test_chat_stream_that_ends_cleanly_without_a_finish_reason_reaches_the_client_as_an_error(
+    gateway: Gateway, family: Family
+) -> None:
+    with wire_server(_upstream(family)) as wire, gateway.scenario() as scenario:
+        model: Final = _deploy(scenario, family, wire)
+        text, call_id = _chat(gateway, model, "ending-clean-eof")
+        assert _chat_text(text) == "".join(_PARTIAL), text
+        assert _finish_reasons(text) == (), text
+        assert len(_chat_errors(text)) == 1, text
+        assert spend_row(call_id)["status"] == "failure"
+
+
+def test_chat_stream_that_sends_done_without_a_finish_reason_reaches_the_client_as_an_error(gateway: Gateway) -> None:
+    with wire_server(_upstream("openai")) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=_DEPLOYMENTS["openai"], api_base=wire.url, api_key=_PROVIDER_KEY)
+        text, call_id = _chat(gateway, model, "ending-done-without-finish")
+        assert _chat_text(text) == "".join(_PARTIAL), text
+        assert _finish_reasons(text) == (), text
+        assert len(_chat_errors(text)) == 1, text
+        assert spend_row(call_id)["status"] == "failure"
+
+
+def test_chat_stream_that_sends_only_done_still_completes_empty(gateway: Gateway) -> None:
+    """An upstream can answer with only [DONE]: Gemini's OpenAI-compatible endpoint did on 2026-10-20 when
+    reasoning used up max_tokens."""
+    with wire_server(_upstream("openai")) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=_DEPLOYMENTS["openai"], api_base=wire.url, api_key=_PROVIDER_KEY)
+        text, _ = _chat(gateway, model, "ending-done-only")
+        assert _chat_text(text) == "", text
+        assert _finish_reasons(text) == ("stop",), text
+        assert _chat_errors(text) == (), text
+        assert text.rstrip().endswith("data: [DONE]"), text
+
+
+@pytest.mark.parametrize("family", ["openai", "anthropic", "gemini", "langgraph", "agentcore"])
+def test_chat_stream_with_a_provider_finish_reason_still_completes(gateway: Gateway, family: Family) -> None:
+    with wire_server(_upstream(family)) as wire, gateway.scenario() as scenario:
+        model: Final = _deploy(scenario, family, wire)
+        text, call_id = _chat(gateway, model, "ending-complete")
+        assert _chat_text(text) == "".join(_PARTIAL), text
+        assert _finish_reasons(text) == ("stop",), text
+        assert _chat_errors(text) == (), text
+        assert text.rstrip().endswith("data: [DONE]"), text
+        assert spend_row(call_id)["status"] == "success"
+
+
+def test_chat_stream_from_a_mock_response_deployment_still_completes(gateway: Gateway) -> None:
+    """mock_completion streams never carry a finish reason, so a mocked deployment must keep its synthetic stop."""
+    with gateway.scenario() as scenario:
+        model: Final = scenario.model(model=_DEPLOYMENTS["openai"], mock_response="".join(_PARTIAL))
+        text, call_id = _chat(gateway, model, "ending-clean-eof")
+        assert _chat_text(text) == "".join(_PARTIAL), text
+        assert _finish_reasons(text) == ("stop",), text
+        assert _chat_errors(text) == (), text
+        assert spend_row(call_id)["status"] == "success"
+
+
+@pytest.mark.parametrize(("ending", "terminal"), [("ending-complete", True), ("ending-clean-eof", False)])
+def test_messages_over_a_chat_bridged_deployment_reports_end_turn_only_for_a_finished_stream(
+    gateway: Gateway, ending: Ending, terminal: bool
+) -> None:
+    with wire_server(_upstream("hosted_vllm")) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=_DEPLOYMENTS["hosted_vllm"], api_base=wire.url, api_key=_PROVIDER_KEY)
+        text, call_id = _post(
+            gateway,
+            "/v1/messages",
+            {"model": model, "max_tokens": 64, "messages": [{"role": "user", "content": _prompt(ending)}]},
+        )
+        types: Final = _event_types(text)
+        stop_reasons: Final = tuple(
+            data["delta"]["stop_reason"]
+            for event, data in sse_events(text)
+            if event == "message_delta" and isinstance(data.get("delta"), dict)
+        )
+        assert types[0] == "message_start", types
+        assert ("message_stop" in types) is terminal, types
+        assert ("error" in types) is not terminal, types
+        assert stop_reasons == (("end_turn",) if terminal else ()), types
+        assert spend_row(call_id)["status"] == ("success" if terminal else "failure")
+
+
+@pytest.mark.parametrize(("ending", "terminal"), [("ending-complete", True), ("ending-clean-eof", False)])
+def test_responses_over_a_chat_bridged_anthropic_deployment_completes_only_a_finished_stream(
+    gateway: Gateway, ending: Ending, terminal: bool
+) -> None:
+    with wire_server(_upstream("anthropic")) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=_DEPLOYMENTS["anthropic"], api_base=wire.url, api_key=_PROVIDER_KEY)
+        text, call_id = _post(gateway, "/v1/responses", {"model": model, "input": _prompt(ending)})
+        types: Final = tuple(string_value(frame.get("type") or "") for frame in data_frames(text))
+        assert types[:1] == ("response.created",), text
+        assert ("response.completed" in types) is terminal, text
+        assert (types[-1] in {"error", "response.failed"}) is not terminal, text
+        assert spend_row(call_id)["status"] == ("success" if terminal else "failure")
+
+
+def _drain_openai_sdk(gateway: Gateway, model: str, sync_mode: bool) -> None:
+    messages: Final = [{"role": "user", "content": _prompt("ending-clean-eof")}]
+    if sync_mode:
+        for _ in openai_client(gateway).chat.completions.create(model=model, messages=messages, stream=True):
+            pass
+        return
+
+    async def drain() -> None:
+        stream: Final = await async_openai_client(gateway).chat.completions.create(
+            model=model, messages=messages, stream=True
+        )
+        async for _ in stream:
+            pass
+
+    asyncio.run(drain())
+
+
+def _drain_anthropic_sdk(gateway: Gateway, model: str, sync_mode: bool) -> None:
+    messages: Final = [{"role": "user", "content": _prompt("ending-clean-eof")}]
+    if sync_mode:
+        for _ in anthropic_client(gateway).messages.create(model=model, max_tokens=64, messages=messages, stream=True):
+            pass
+        return
+
+    async def drain() -> None:
+        stream: Final = await async_anthropic_client(gateway).messages.create(
+            model=model, max_tokens=64, messages=messages, stream=True
+        )
+        async for _ in stream:
+            pass
+
+    asyncio.run(drain())
+
+
+@pytest.mark.parametrize("sync_mode", [True, False])
+def test_openai_sdk_stream_cut_before_a_finish_reason_raises(gateway: Gateway, sync_mode: bool) -> None:
+    with wire_server(_upstream("openai")) as wire, gateway.scenario() as scenario:
+        model: Final = _deploy(scenario, "openai", wire)
+        with pytest.raises(openai.APIError, match="Stream ended before a finish_reason was received"):
+            _drain_openai_sdk(gateway, model, sync_mode)
+
+
+@pytest.mark.parametrize("sync_mode", [True, False])
+def test_anthropic_sdk_messages_stream_cut_before_a_finish_reason_raises(gateway: Gateway, sync_mode: bool) -> None:
+    with wire_server(_upstream("hosted_vllm")) as wire, gateway.scenario() as scenario:
+        model: Final = _deploy(scenario, "hosted_vllm", wire)
+        with pytest.raises(anthropic.APIError, match="Stream ended before a finish_reason was received"):
+            _drain_anthropic_sdk(gateway, model, sync_mode)
+
+
+def test_responses_stream_cut_on_its_deployment_finishes_on_the_request_fallback(gateway: Gateway) -> None:
+    with (
+        wire_server(_upstream("anthropic")) as primary,
+        wire_server(_upstream("anthropic", "ending-complete")) as backup,
+        gateway.scenario() as scenario,
+    ):
+        model: Final = _deploy(scenario, "anthropic", primary)
+        fallback: Final = _deploy(scenario, "anthropic", backup)
+        text, _ = _post(
+            gateway, "/v1/responses", {"model": model, "input": _prompt("ending-clean-eof"), "fallbacks": [fallback]}
+        )
+        types: Final = tuple(string_value(frame.get("type") or "") for frame in data_frames(text))
+        assert "response.completed" in types and "response.failed" not in types, text
+        assert (len(posted_targets(primary)), len(posted_targets(backup))) == (1, 1), text
+
+
+def test_chat_stream_cut_after_its_content_is_not_replayed_on_the_request_fallback(gateway: Gateway) -> None:
+    with (
+        wire_server(_upstream("openai")) as primary,
+        wire_server(_upstream("openai", "ending-complete")) as backup,
+        gateway.scenario() as scenario,
+    ):
+        model: Final = _deploy(scenario, "openai", primary)
+        fallback: Final = _deploy(scenario, "openai", backup)
+        text, _ = _post(
+            gateway,
+            "/v1/chat/completions",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": _prompt("ending-clean-eof")}],
+                "fallbacks": [fallback],
+            },
+        )
+        assert _chat_text(text) == "".join(_PARTIAL), text
+        assert len(_chat_errors(text)) == 1, text
+        assert posted_targets(backup) == (), text
+
+
+Route = Literal["/v1/chat/completions", "/v1/messages", "/v1/responses"]
+
+
+def _route_body(route: Route, model: str, ending: Ending) -> Mapping[str, JsonValue]:
+    match route:
+        case "/v1/chat/completions":
+            return {"model": model, "messages": [{"role": "user", "content": _prompt(ending)}]}
+        case "/v1/messages":
+            return {"model": model, "max_tokens": 64, "messages": [{"role": "user", "content": _prompt(ending)}]}
+        case "/v1/responses":
+            return {"model": model, "input": _prompt(ending)}
+
+
+def _finished(route: Route, text: str) -> bool:
+    match route:
+        case "/v1/chat/completions":
+            return _finish_reasons(text) == ("stop",) and _chat_errors(text) == ()
+        case "/v1/messages":
+            return "message_stop" in _event_types(text) and "error" not in _event_types(text)
+        case "/v1/responses":
+            return "response.completed" in tuple(string_value(frame.get("type") or "") for frame in data_frames(text))
+
+
+def _stream_outcome(gateway: Gateway, model: str, route: Route, ending: Ending) -> tuple[bool, str]:
+    text, call_id = _post(gateway, route, _route_body(route, model, ending))
+    return _finished(route, text), call_id
+
+
+_BRIDGED_FAMILY: Final[Mapping[Route, Family]] = {
+    "/v1/chat/completions": "openai",
+    "/v1/messages": "hosted_vllm",
+    "/v1/responses": "anthropic",
+}
+
+
+def test_concurrent_burst_ends_each_stream_by_whether_its_provider_finished_it(gateway: Gateway) -> None:
+    endings: Final[tuple[Ending, ...]] = ("ending-complete", "ending-clean-eof")
+    cases: Final = tuple(itertools.product(_BRIDGED_FAMILY, endings, range(5)))
+    with ExitStack() as wires, gateway.scenario() as scenario:
+        models: Final = {
+            route: _deploy(scenario, family, wires.enter_context(wire_server(_upstream(family))))
+            for route, family in _BRIDGED_FAMILY.items()
+        }
+        with ThreadPoolExecutor(max_workers=12) as pool:
+            outcomes: Final = tuple(
+                pool.map(lambda case: _stream_outcome(gateway, models[case[0]], case[0], case[1]), cases)
+            )
+        assert tuple(finished for finished, _ in outcomes) == tuple(
+            ending == "ending-complete" for _, ending, _ in cases
+        ), outcomes
+        assert tuple(spend_row(call_id)["status"] for _, call_id in outcomes) == tuple(
+            "success" if ending == "ending-complete" else "failure" for _, ending, _ in cases
+        )
+
+
+_SPEND_LOG_INSERT: Final = b'INSERT INTO "public"."LiteLLM_SpendLogs"'
+
+
+def _spend_statuses(database_url: str | None) -> Counter[tuple[str, str]]:
+    rows: Final = read_rows('SELECT litellm_call_id, status FROM "LiteLLM_SpendLogs"', (), database_url=database_url)
+    return Counter((string_value(row["litellm_call_id"]), string_value(row["status"])) for row in rows)
+
+
+def test_spend_log_outage_mid_burst_keeps_each_stream_ending_and_lands_every_row_once(
+    gateway: Gateway, tmp_path: Path
+) -> None:
+    endings: Final[tuple[Ending, ...]] = ("ending-complete", "ending-clean-eof")
+    cases: Final = tuple(itertools.product(_BRIDGED_FAMILY, endings))
+    with (
+        scratch_database() as database_url,
+        database_relay(database_url, _SPEND_LOG_INSERT) as (relay, relayed_url),
+        owned_proxy(gateway, tmp_path, {"DATABASE_URL": plaintext_database_url(relayed_url)}, workers=2) as candidate,
+        ExitStack() as wires,
+        candidate.scenario() as scenario,
+        ThreadPoolExecutor(max_workers=len(cases)) as pool,
+    ):
+        models: Final = {
+            route: _deploy(scenario, family, wires.enter_context(wire_server(_upstream(family))))
+            for route, family in _BRIDGED_FAMILY.items()
+        }
+
+        def burst() -> tuple[tuple[bool, str], ...]:
+            return tuple(pool.map(lambda case: _stream_outcome(candidate, models[case[0]], case[0], case[1]), cases))
+
+        relay.arm()
+        before: Final = burst()
+        assert relay.tripped.wait(90), "no spend log write reached the database after the first burst"
+        down: Final = not relay.reconnected.is_set()
+        during: Final = burst()
+        assert relay.reconnected.wait(90), "the proxy never reconnected to the database after the outage"
+        assert down and relay.refused >= 1, relay.refused
+        finished: Final = tuple(ending == "ending-complete" for _, ending in cases)
+        assert tuple(done for done, _ in before + during) == finished * 2, (before, during)
+        expected: Final = Counter(
+            (call_id, "success" if ending == "ending-complete" else "failure")
+            for (_, ending), (_, call_id) in zip(cases * 2, before + during, strict=True)
+        )
+        landed: Final = eventually(
+            lambda: _spend_statuses(database_url),
+            lambda rows: {call_id for call_id, _ in expected} <= {call_id for call_id, _ in rows},
+            seconds=120,
+        )
+        assert landed == expected
+
+
+def _attempt(gateway: Gateway, model: str, route: Route, ending: Ending) -> tuple[bool, str] | None:
+    """One stream's outcome, or None when the worker serving it was killed before the proxy finished the reply"""
+    try:
+        return _stream_outcome(gateway, model, route, ending)
+    except httpx.TransportError:
+        return None
+
+
+def test_worker_killed_mid_burst_leaves_the_proxy_ending_each_stream_by_whether_its_provider_finished_it(
+    gateway: Gateway, tmp_path: Path
+) -> None:
+    endings: Final[tuple[Ending, ...]] = ("ending-complete", "ending-clean-eof")
+    cases: Final = tuple(itertools.product(_BRIDGED_FAMILY, endings, range(4)))
+    probes: Final = tuple(itertools.product(_BRIDGED_FAMILY, endings))
+    with (
+        owned_proxy_process(gateway, tmp_path, {}, workers=2) as owned,
+        ExitStack() as wires,
+        owned.gateway.scenario() as scenario,
+    ):
+        candidate: Final = owned.gateway
+        models: Final = {
+            route: _deploy(scenario, family, wires.enter_context(wire_server(_upstream(family))))
+            for route, family in _BRIDGED_FAMILY.items()
+        }
+        workers: Final = eventually(owned.worker_pids, lambda pids: len(pids) == 2, seconds=60)
+        with owned.replacing_worker(min(workers)) as kill, ThreadPoolExecutor(max_workers=len(cases)) as pool:
+            futures: Final = [
+                pool.submit(_attempt, candidate, models[route], route, ending) for route, ending, _ in cases
+            ]
+            kill()
+            attempts: Final = tuple(future.result() for future in futures)
+        answered: Final = tuple(
+            (outcome[0], ending == "ending-complete")
+            for (_, ending, _), outcome in zip(cases, attempts, strict=True)
+            if outcome is not None
+        )
+        assert all(finished == expected for finished, expected in answered), attempts
+        after: Final = tuple(_stream_outcome(candidate, models[route], route, ending) for route, ending in probes)
+        assert tuple(finished for finished, _ in after) == tuple(ending == "ending-complete" for _, ending in probes)
+        statuses: Final = eventually(
+            lambda: _spend_statuses(candidate.database_url),
+            lambda rows: {call_id for _, call_id in after} <= {call_id for call_id, _ in rows},
+            seconds=120,
+        )
+        assert tuple(
+            statuses[(call_id, "success" if ending == "ending-complete" else "failure")]
+            for (_, ending), (_, call_id) in zip(probes, after, strict=True)
+        ) == (1,) * len(probes)

tests/unit/batches/test_batch_utils.py

@@ -3436,3 +3436,57 @@ def test_invalid_batch_envelopes_preserve_recoverable_cost(
     assert result.usage.prompt_tokens == (200 if include_valid else 0)
     assert result.usage.completion_tokens == (40 if include_valid else 0)
     assert (result.successful_requests, result.failed_requests) == (2 if include_valid else 0, 1)
+
+
+def test_unpriced_batch_retains_reported_usage_by_model_and_marks_missing_usage():
+    rows: Final = [
+        _success_row(model="unregistered-batch-model-a", usage=_usage(4, 2)),
+        _success_row(model="unregistered-batch-model-a", usage=_usage(3, 1)),
+        _success_row(model="unregistered-batch-model-b", usage=_usage(5, 2)),
+        {"response": {"status_code": 200, "body": {"model": "unregistered-batch-model-c"}}},
+    ]
+    result: Final = bu.batch_cost_and_usage(rows, "openai")
+    captured: Final = result.usage.model_dump()
+    assert (result.usage.prompt_tokens, result.usage.completion_tokens) == (12, 5)
+    assert result.cost == 0
+    by_model: Final = captured["batch_usage_by_model"]
+    assert by_model["unregistered-batch-model-a"]["total_tokens"] == 10
+    assert by_model["unregistered-batch-model-b"]["total_tokens"] == 7
+    assert by_model["unregistered-batch-model-c"] is None
+    assert captured["batch_unknown_usage_requests"] == 1
+    per_request: Final = captured["batch_usage_by_request"]
+    assert len(per_request) == len(rows)
+    assert per_request[-1] == {"model": "unregistered-batch-model-c", "usage": None}
+    reordered: Final = bu.batch_cost_and_usage(list(reversed(rows)), "openai")
+    assert reordered.usage.model_dump()["batch_usage_by_request"] == per_request
+
+
+def test_batch_ocr_preserves_additive_page_quantities_without_changing_cost():
+    rows: Final = [
+        {
+            "response": {
+                "status_code": 200,
+                "body": {
+                    "model": "ocr-quantity-control",
+                    "usage_info": {"pages_processed": pages, "doc_size_bytes": size},
+                },
+            }
+        }
+        for pages, size in ((2, 100), (3, 200))
+    ]
+    result: Final = bu.batch_cost_and_usage(rows, "mistral", model_info={"ocr_cost_per_page_batches": 0.01})
+    captured: Final = result.usage.model_dump()["batch_usage_by_model"]["ocr-quantity-control"]
+    assert captured["pages_processed"] == 5
+    assert captured["doc_size_bytes"] == 300
+    assert result.cost == pytest.approx(5 * 0.01)
+    assert sorted(row["usage"]["pages_processed"] for row in result.usage.model_dump()["batch_usage_by_request"]) == [2, 3]
+
+
+def test_native_vertex_capture_retains_unknown_models_without_changing_legacy_counts():
+    rows: Final = [{"request": {}, "response": {"modelVersion": "missing-usage-model", "candidates": []}}]
+    result: Final = bu.calculate_vertex_ai_batch_cost_and_usage(rows)
+    captured: Final = result.usage.model_dump()
+    assert captured["batch_usage_by_model"] == {"missing-usage-model": None}
+    assert captured["batch_usage_by_request"] == ({"model": "missing-usage-model", "usage": None},)
+    assert captured["batch_unknown_usage_requests"] == 1
+    assert (result.cost, result.successful_requests, result.failed_requests) == (0, 0, 1)

tests/unit/integrations/test_anthropic_cache_control_hook.py

@@ -1116,7 +1116,7 @@ def test_cache_control_hook_counts_tool_call_cache_controls():
     assert AnthropicCacheControlHook.count_request_cache_breakpoints([message]) == 3
 
 
-def test_cache_control_hook_counts_tool_call_marks_except_answered_server_tool_calls():
+def test_cache_control_hook_counts_tool_call_marks_except_server_tool_calls_the_client_did_not_answer():
     message: Final[AllMessageValues] = {
         "role": "assistant",
         "content": None,
@@ -1170,7 +1170,7 @@ def test_cache_control_hook_counts_tool_call_marks_except_answered_server_tool_c
         },
     }
 
-    assert AnthropicCacheControlHook.count_request_cache_breakpoints([message]) == 2
+    assert AnthropicCacheControlHook.count_request_cache_breakpoints([message]) == 1
 
 
 _SEARCH_ANSWER: Final = {"role": "tool", "tool_call_id": "srvtoolu_01Search", "content": "found get_weather"}
@@ -1182,6 +1182,12 @@ _SEARCH_CALL: Final = {
     "function": {"name": "tool_search_tool_bm25", "arguments": "{}"},
     "cache_control": {"type": "ephemeral"},
 }
+_UNRUN_WEB_SEARCH_CALL: Final = {
+    "id": "srvtoolu_01Web",
+    "type": "function",
+    "function": {"name": "web_search", "arguments": "{}"},
+    "cache_control": {"type": "ephemeral"},
+}
 _WEATHER_CALL: Final = {
     "id": "toolu_01Weather",
     "type": "function",
@@ -1238,6 +1244,8 @@ def _replayed_search_call(*tool_calls: dict[str, object]) -> AllMessageValues:
             (_SEARCH_CALL,), (_SEARCH_ANSWER, _GO_ON), False, 0, id="client_answer_replaced_by_the_kept_result"
         ),
         pytest.param((_SEARCH_CALL,), (_GO_ON,), True, 0, id="placeholder_replaced_by_the_kept_result"),
+        pytest.param((_UNRUN_WEB_SEARCH_CALL,), (), False, 0, id="trailing_unrun_search_replayed_without_its_mark"),
+        pytest.param((_UNRUN_WEB_SEARCH_CALL,), (_GO_ON,), False, 0, id="unrun_search_dropped_before_a_new_message"),
         pytest.param(
             (_SEARCH_CALL, _WEATHER_CALL),
             (_SEARCH_ANSWER,),

tests/unit/litellm_core_utils/llm_response_utils/test_convert_to_streaming_response.py

@@ -563,7 +563,7 @@ async def test_a_cached_answer_streams_its_tool_call_after_the_reasoning_and_the
     assert reasoning and max(reasoning) < min(text)
     assert called == [max(text) + 1]
     assert deltas[called[0]]["tool_calls"] == [tool_call]
-    assert deltas[-1]["finish"] == "stop"
+    assert [delta["finish"] for delta in deltas if delta["finish"]] == ["tool_calls"]
 
 
 @pytest.mark.asyncio
@@ -586,7 +586,7 @@ async def test_a_cached_tool_turn_without_text_streams_the_reasoning_before_the_
     assert "".join(str(deltas[i]["reasoning_content"]) for i in reasoning) == "I need the basket totals"
     assert reasoning and called and max(reasoning) < min(called)
     assert [deltas[i]["tool_calls"] for i in called] == [[tool_call]]
-    assert deltas[-1]["finish"] == "stop"
+    assert [delta["finish"] for delta in deltas if delta["finish"]] == ["tool_calls"]
 
 
 @pytest.mark.asyncio
@@ -610,4 +610,4 @@ async def test_a_cached_tool_turn_keeps_its_whitespace_only_text_between_the_rea
     assert "".join(str(deltas[i]["content"]) for i in text) == whitespace
     assert reasoning and text and called and max(reasoning) < min(text) and max(text) < min(called)
     assert [deltas[i]["tool_calls"] for i in called] == [[tool_call]]
-    assert deltas[-1]["finish"] == "stop"
+    assert [delta["finish"] for delta in deltas if delta["finish"]] == ["tool_calls"]

tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py

@@ -2001,6 +2001,12 @@ class TestRequestContainsImageContent:
             [{"role": "user", "content": [{"type": "input_audio", "input_audio": {"data": "x"}}]}],
             [{"role": "user", "content": [{"type": "tool_result", "content": [{"type": "text", "text": "ok"}]}]}],
             [{"role": "user", "content": None}],
+            [{"role": "user", "content": [None, "text", {"type": []}]}],
+            ["malformed"],
+            [None, 5],
+            "malformed",
+            5,
+            None,
             [],
         ],
     )

tests/unit/litellm_core_utils/test_litellm_logging.py

@@ -2946,6 +2946,44 @@ async def test_async_success_handler_runs_async_callbacks_in_the_post_response_p
     assert in_post_response_phase() is False, "the phase must not leak into the request task"
 
 
+@pytest.mark.parametrize("stream", [True, False], ids=["streaming", "non_streaming"])
+def test_reused_logging_obj_does_not_report_the_previous_attempts_first_token(logging_obj, stream: bool):
+    """A fallback reuses the request's logging object but runs in its own window.
+
+    When the first attempt already recorded a first-token time, the fallback must not
+    report it. Otherwise ``completion_start_time`` lands before the attempt's
+    ``start_time``, the callbacks see a negative TTFT, and latency-based routing caches
+    a negative sample that empties its buffer window.
+    """
+    logging_obj.stream = stream
+    logging_obj.model_call_details["litellm_params"] = {}
+    logging_obj.litellm_params = {}
+
+    first_attempt_start: Final = datetime.datetime(2026, 1, 1, 12, 0, 0)
+    logging_obj.update_completion_start_time(first_attempt_start + datetime.timedelta(seconds=1))
+    fallback_start: Final = first_attempt_start + datetime.timedelta(seconds=600)
+    fallback_end: Final = fallback_start + datetime.timedelta(seconds=5)
+
+    observed: dict[str, object] = {}
+
+    class Recorder(CustomLogger):
+        def log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
+            observed["completion_start_time"] = kwargs.get("completion_start_time")
+
+    with patch.object(logging_obj, "get_combined_callback_list", return_value=[Recorder()]):
+        logging_obj.success_handler(
+            result=_success_response(),
+            start_time=fallback_start,
+            end_time=fallback_end,
+        )
+
+    completion_start_time: Final = observed["completion_start_time"]
+    assert isinstance(completion_start_time, datetime.datetime)
+    assert completion_start_time >= fallback_start, (
+        "the fallback attempt reported a first-token time that predates its own start"
+    )
+
+
 def test_is_sync_litellm_request():
     assert LitellmLogging.is_sync_litellm_request({}) is True
     assert LitellmLogging.is_sync_litellm_request({"acompletion": True}) is False
@@ -14231,6 +14269,71 @@ def test_native_messages_body_carries_the_reported_list_cost_into_the_priced_res
     assert headers.get("llm_provider-x-litellm-response-cost") == expected_header
 
 
+_NATIVE_PRICED_MESSAGE: Final = {
+    "id": "msg_native",
+    "type": "message",
+    "role": "assistant",
+    "model": "claude-haiku-4-5",
+    "content": [{"type": "text", "text": "answer"}],
+    "stop_reason": "end_turn",
+    "stop_sequence": None,
+    "usage": {"input_tokens": 1200, "output_tokens": 300},
+}
+
+
+def _native_anthropic_messages_logging_obj() -> LitellmLogging:
+    logging_obj: Final = _anthropic_messages_logging_obj()
+    logging_obj.stream = False
+    logging_obj.optional_params = {}
+    logging_obj.model = "claude-haiku-4-5"
+    logging_obj.model_call_details["custom_llm_provider"] = "anthropic"
+    return logging_obj
+
+
+@pytest.mark.parametrize(
+    "served_body",
+    (
+        {**_NATIVE_PRICED_MESSAGE, "content": None},
+        {key: value for key, value in _NATIVE_PRICED_MESSAGE.items() if key != "content"},
+    ),
+    ids=("content-null", "content-missing"),
+)
+def test_a_native_messages_body_without_content_is_priced_from_its_usage(served_body: dict[str, object]) -> None:
+    """Regression: #48254 converted every native /v1/messages body before pricing it, and the conversion
+    raised on a 200 whose ``content`` was null or missing, turning the served 200 into a 500."""
+    with_content: Final = _native_anthropic_messages_logging_obj().response_cost_calculator(
+        result=_NATIVE_PRICED_MESSAGE
+    )
+    assert with_content is not None and with_content > 0
+
+    without_content: Final = _native_anthropic_messages_logging_obj().response_cost_calculator(result=served_body)
+
+    assert without_content == with_content
+
+
+def _fast_mode_native_logging_obj(requested_speed: str | None) -> LitellmLogging:
+    logging_obj: Final = _native_anthropic_messages_logging_obj()
+    logging_obj.model = "claude-opus-5"
+    logging_obj.model_call_details["model"] = "claude-opus-5"
+    logging_obj.optional_params = {"speed": requested_speed} if requested_speed else {}
+    return logging_obj
+
+
+def test_a_native_messages_body_without_a_reported_speed_bills_the_requested_speed() -> None:
+    """A fast-mode request whose upstream usage block omits ``speed`` is still billed at the fast rate,
+    the same rate the spend row applies from the request's ``speed``."""
+    body: Final = {**_NATIVE_PRICED_MESSAGE, "model": "claude-opus-5"}
+    standard: Final = _fast_mode_native_logging_obj(None).response_cost_calculator(result=body)
+    reported_fast: Final = _fast_mode_native_logging_obj(None).response_cost_calculator(
+        result={**body, "usage": {**_NATIVE_PRICED_MESSAGE["usage"], "speed": "fast"}}
+    )
+    assert standard is not None and reported_fast is not None and reported_fast > standard
+
+    requested_fast: Final = _fast_mode_native_logging_obj("fast").response_cost_calculator(result=body)
+
+    assert requested_fast == reported_fast
+
+
 def test_record_native_bedrock_guardrail_bills_when_the_metadata_is_none() -> None:
     import datetime
 

tests/unit/litellm_core_utils/test_streaming_handler.py

@@ -32,8 +32,12 @@ from litellm.litellm_core_utils.streaming_handler import (
     _ProviderChunkEarlyReturn,
     _ProviderChunkParsed,
 )
+from litellm.llms.anthropic.chat.handler import ModelResponseIterator as AnthropicModelResponseIterator
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
+from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
+    ModelResponseIterator as GeminiModelResponseIterator,
+)
 from litellm.responses.mcp.chat_completions_handler import _add_mcp_metadata_to_response
 from litellm.types.utils import (
     ChoiceLogprobs,
@@ -572,11 +576,6 @@ _GET_WEATHER_CALL: Final = {
             "<think>a</think>b<think>c</think>d",
             0,
         ),
-        (
-            (_stream_chunk({"reasoning_content": "Still thinking"}),),
-            "<think>Still thinking</think>",
-            0,
-        ),
         (
             (_stream_chunk({"content": "hi"}), _stream_chunk({}, "stop")),
             "hi",
@@ -588,7 +587,6 @@ _GET_WEATHER_CALL: Final = {
         "reasoning_then_function_call",
         "reasoning_cut_by_length",
         "reasoning_after_answer",
-        "stream_ends_mid_reasoning",
         "answer_without_reasoning",
     ],
 )
@@ -622,6 +620,26 @@ async def test_merged_reasoning_stream_keeps_think_tags_balanced(
     )
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("sync_mode", [True, False])
+async def test_merged_reasoning_stream_cut_mid_reasoning_fails_instead_of_closing_the_think_block(
+    logging_obj: Logging, sync_mode: bool
+) -> None:
+    response: Final = CustomStreamWrapper(
+        completion_stream=ModelResponseListIterator(
+            model_responses=[_stream_chunk({"reasoning_content": "Still thinking"})]
+        ),
+        model="deepseek-reasoner",
+        logging_obj=logging_obj,
+    )
+    response.merge_reasoning_content_in_choices = True
+
+    with pytest.raises(MidStreamFallbackError) as excinfo:
+        await _collect(response, sync_mode)
+
+    assert excinfo.value.generated_content == "<think>Still thinking"
+
+
 def test_strip_sse_data_from_chunk():
     """Test the static method that strips 'data: ' prefix from SSE chunks"""
     # Test with string inputs
@@ -2270,6 +2288,242 @@ async def test_mid_stream_error_keeps_credentials_in_the_api_base_out(routed_api
     assert "deployment-secret" not in debug_info
 
 
+def _incomplete_openai_chat_chunk() -> ModelResponseStream:
+    return ModelResponseStream(
+        id="chatcmpl-incomplete-eof",
+        created=1,
+        model="gpt-5.6",
+        object="chat.completion.chunk",
+        choices=[
+            StreamingChoices(
+                index=0,
+                delta=Delta(content='{"findings":[{"title":"unfinished', role="assistant"),
+                finish_reason=None,
+            )
+        ],
+    )
+
+
+@pytest.mark.parametrize("custom_llm_provider", ["openai", "azure", "hosted_vllm"])
+@pytest.mark.asyncio
+async def test_async_openai_compatible_chat_clean_eof_without_finish_reason_raises_midstream_fallback(
+    logging_obj: Logging, custom_llm_provider: str
+) -> None:
+    """An OpenAI-compatible stream without a provider finish reason must fail at clean EOF."""
+
+    logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider
+    response: Final = CustomStreamWrapper(
+        completion_stream=_as_async(iter([_incomplete_openai_chat_chunk()])),
+        model="gpt-5.6",
+        logging_obj=logging_obj,
+        custom_llm_provider=custom_llm_provider,
+        stream_options={"include_usage": True},
+    )
+
+    first_chunk: Final = await response.__anext__()
+    with pytest.raises(MidStreamFallbackError) as excinfo:
+        await response.__anext__()
+
+    assert first_chunk.choices[0].delta.content == '{"findings":[{"title":"unfinished'
+    assert excinfo.value.generated_content == '{"findings":[{"title":"unfinished'
+    assert excinfo.value.is_pre_first_chunk is False
+    assert response.received_finish_reason is None
+
+
+@pytest.mark.parametrize("custom_llm_provider", ["openai", "azure", "hosted_vllm"])
+def test_sync_openai_compatible_chat_clean_eof_without_finish_reason_raises_midstream_fallback(
+    logging_obj: Logging, custom_llm_provider: str
+) -> None:
+    """The synchronous OpenAI-compatible stream must fail at clean EOF too."""
+
+    logging_obj.model_call_details["custom_llm_provider"] = custom_llm_provider
+    response: Final = CustomStreamWrapper(
+        completion_stream=iter([_incomplete_openai_chat_chunk()]),
+        model="gpt-5.6",
+        logging_obj=logging_obj,
+        custom_llm_provider=custom_llm_provider,
+        stream_options={"include_usage": True},
+    )
+
+    first_chunk: Final = next(response)
+    with pytest.raises(MidStreamFallbackError) as excinfo:
+        next(response)
+
+    assert first_chunk.choices[0].delta.content == '{"findings":[{"title":"unfinished'
+    assert excinfo.value.generated_content == '{"findings":[{"title":"unfinished'
+    assert excinfo.value.is_pre_first_chunk is False
+    assert response.received_finish_reason is None
+
+
+async def _collect(wrapper: CustomStreamWrapper, sync_mode: bool) -> list[ModelResponseStream]:
+    if sync_mode:
+        return list(wrapper)
+    return [chunk async for chunk in wrapper]
+
+
+@pytest.mark.parametrize("custom_llm_provider", ["openai", "azure", "hosted_vllm"])
+@pytest.mark.parametrize("sync_mode", [True, False])
+@pytest.mark.asyncio
+async def test_stream_that_sends_no_chunks_keeps_the_default_stop(
+    logging_obj: Logging, custom_llm_provider: str, sync_mode: bool
+) -> None:
+    """Gemini's OpenAI-compatible endpoint answers with only [DONE] when reasoning uses up max_tokens."""
+
+    wrapper: Final = CustomStreamWrapper(
+        completion_stream=iter(()) if sync_mode else _as_async(iter(())),
+        model="gemini-2.5-flash",
+        logging_obj=logging_obj,
+        custom_llm_provider=custom_llm_provider,
+        stream_options={"include_usage": True},
+    )
+
+    received: Final = await _collect(wrapper, sync_mode)
+
+    assert [c.choices[0].finish_reason for c in received if c.choices and c.choices[0].finish_reason] == ["stop"]
+
+
+_ANTHROPIC_PARTIAL_EVENTS: Final[tuple[dict[str, object], ...]] = (
+    {
+        "type": "message_start",
+        "message": {
+            "id": "msg_incomplete_eof",
+            "type": "message",
+            "role": "assistant",
+            "model": "claude-sonnet-4-5",
+            "content": [],
+            "stop_reason": None,
+            "usage": {"input_tokens": 9, "output_tokens": 1},
+        },
+    },
+    {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}},
+    {
+        "type": "content_block_delta",
+        "index": 0,
+        "delta": {"type": "text_delta", "text": '{"findings":[{"title":"unfinished'},
+    },
+)
+_ANTHROPIC_TERMINAL_EVENTS: Final[tuple[dict[str, object], ...]] = (
+    {"type": "content_block_stop", "index": 0},
+    {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 7}},
+    {"type": "message_stop"},
+)
+
+
+def _anthropic_sse_lines(events: Sequence[Mapping[str, object]]) -> list[str]:
+    return [line for event in events for line in (f"event: {event['type']}", f"data: {json.dumps(event)}", "")]
+
+
+def _anthropic_stream(
+    events: Sequence[Mapping[str, object]], sync_mode: bool, logging_obj: Logging
+) -> CustomStreamWrapper:
+    lines: Final = _anthropic_sse_lines(events)
+
+    return CustomStreamWrapper(
+        completion_stream=AnthropicModelResponseIterator(
+            streaming_response=iter(lines) if sync_mode else _as_async(iter(lines)), sync_stream=sync_mode
+        ),
+        model="claude-sonnet-4-5",
+        custom_llm_provider="anthropic",
+        logging_obj=logging_obj,
+    )
+
+
+@pytest.mark.parametrize("sync_mode", [True, False])
+@pytest.mark.asyncio
+async def test_anthropic_stream_that_ends_before_message_delta_fails_with_the_eof_message(
+    sync_mode: bool, logging_obj: Logging
+) -> None:
+    wrapper: Final = _anthropic_stream(_ANTHROPIC_PARTIAL_EVENTS, sync_mode, logging_obj)
+
+    with pytest.raises(MidStreamFallbackError) as excinfo:
+        await _collect(wrapper, sync_mode)
+
+    assert excinfo.value.generated_content == '{"findings":[{"title":"unfinished'
+    assert excinfo.value.is_pre_first_chunk is False
+    assert isinstance(excinfo.value.original_exception, litellm.APIConnectionError)
+    assert "AnthropicException - Stream ended before a finish_reason was received" in str(excinfo.value)
+    assert wrapper.received_finish_reason is None
+
+
+@pytest.mark.parametrize("sync_mode", [True, False])
+@pytest.mark.asyncio
+async def test_anthropic_stream_with_message_stop_ends_with_its_finish_reason(
+    sync_mode: bool, logging_obj: Logging
+) -> None:
+    wrapper: Final = _anthropic_stream(
+        (*_ANTHROPIC_PARTIAL_EVENTS, *_ANTHROPIC_TERMINAL_EVENTS), sync_mode, logging_obj
+    )
+
+    received: Final = await _collect(wrapper, sync_mode)
+
+    assert [c.choices[0].finish_reason for c in received if c.choices and c.choices[0].finish_reason] == ["stop"]
+    assert "".join(c.choices[0].delta.content or "" for c in received if c.choices) == (
+        '{"findings":[{"title":"unfinished'
+    )
+
+
+@pytest.mark.parametrize("sync_mode", [True, False])
+@pytest.mark.asyncio
+async def test_generic_chunk_stream_without_a_finish_reason_keeps_the_default_stop(
+    sync_mode: bool, logging_obj: Logging
+) -> None:
+    """Providers that stream GenericStreamingChunks and never mark the last one finished still end with stop,
+    but the logged response built from ``chunks`` never gains that synthetic stop."""
+    chunks: Final = [
+        {"text": text, "tool_use": None, "is_finished": False, "finish_reason": "", "usage": None, "index": 0}
+        for text in ("partial", " reply")
+    ]
+    wrapper: Final = CustomStreamWrapper(
+        completion_stream=ModelResponseListIterator(model_responses=chunks),
+        model="sagemaker/tgi-endpoint",
+        custom_llm_provider="sagemaker",
+        logging_obj=logging_obj,
+    )
+
+    received: Final = await _collect(wrapper, sync_mode)
+
+    assert [c.choices[0].finish_reason for c in received if c.choices and c.choices[0].finish_reason] == ["stop"]
+    assert "".join(c.choices[0].delta.content or "" for c in received if c.choices) == "partial reply"
+    assert all(not (c.choices and c.choices[0].finish_reason) for c in wrapper.chunks)
+
+
+@pytest.mark.parametrize("sync_mode", [True, False])
+@pytest.mark.asyncio
+async def test_gemini_stream_with_a_usage_chunk_after_its_finish_reason_still_completes(
+    sync_mode: bool, logging_obj: Logging
+) -> None:
+    """Gemini marks every chunk unfinished, so its finishReason only shows up mid-stream before the usage chunk."""
+    lines: Final = [
+        f"data: {json.dumps(event)}"
+        for event in (
+            {"candidates": [{"content": {"parts": [{"text": "Hello"}], "role": "model"}, "index": 0}]},
+            {
+                "candidates": [
+                    {"content": {"parts": [{"text": " there"}], "role": "model"}, "finishReason": "STOP", "index": 0}
+                ]
+            },
+            {"usageMetadata": {"promptTokenCount": 3, "candidatesTokenCount": 2, "totalTokenCount": 5}},
+        )
+    ]
+
+    logging_obj.update_environment_variables(
+        model="gemini-2.5-flash", user=None, optional_params={}, litellm_params={"litellm_call_id": "gemini-tail"}
+    )
+    iterator: Final = GeminiModelResponseIterator(
+        streaming_response=iter(lines) if sync_mode else _as_async(iter(lines)), sync_stream=sync_mode, logging_obj=logging_obj
+    )
+    wrapper: Final = CustomStreamWrapper(
+        completion_stream=iter(iterator) if sync_mode else iterator,
+        model="gemini-2.5-flash",
+        custom_llm_provider="gemini",
+        logging_obj=logging_obj,
+    )
+
+    received: Final = await _collect(wrapper, sync_mode)
+
+    assert "".join(c.choices[0].delta.content or "" for c in received if c.choices) == "Hello there"
+    assert [c.choices[0].finish_reason for c in received if c.choices and c.choices[0].finish_reason] == ["stop"]
+
 
 def test_streaming_handler_with_created_time_propagation(
     initialized_custom_stream_wrapper: CustomStreamWrapper, logging_obj: Logging
@@ -3638,11 +3892,8 @@ def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj):
 
 
 @pytest.mark.parametrize("sync_mode", [True, False])
-@pytest.mark.parametrize("ends_with_finish_chunk", [True, False])
 @pytest.mark.asyncio
-async def test_stream_without_provider_usage_falls_back_to_the_token_estimate(
-    sync_mode: bool, ends_with_finish_chunk: bool
-) -> None:
+async def test_stream_without_provider_usage_falls_back_to_the_token_estimate(sync_mode: bool) -> None:
     model: Final = "gpt-4o"
     messages: Final = [{"role": "user", "content": "Write a long sentence about a fox. " * 30}]
     text: Final = "The quick brown fox jumps over the lazy dog. " * 20
@@ -3659,9 +3910,7 @@ async def test_stream_without_provider_usage_falls_back_to_the_token_estimate(
         choices=[StreamingChoices(index=0, delta=Delta(content=""), finish_reason="stop")],
     )
     wrapper: Final = CustomStreamWrapper(
-        completion_stream=ModelResponseListIterator(
-            model_responses=[content_chunk, finish_chunk] if ends_with_finish_chunk else [content_chunk]
-        ),
+        completion_stream=ModelResponseListIterator(model_responses=[content_chunk, finish_chunk]),
         model=model,
         custom_llm_provider="openai",
         logging_obj=Logging(
@@ -3700,6 +3949,55 @@ async def test_stream_without_provider_usage_falls_back_to_the_token_estimate(
     assert litellm.completion_cost(completion_response=assembled) == pytest.approx(spend)
 
 
+@pytest.mark.parametrize("sync_mode", [True, False])
+@pytest.mark.asyncio
+async def test_stream_cut_before_a_finish_chunk_bills_the_token_estimate_on_its_failure(sync_mode: bool) -> None:
+    model: Final = "gpt-4o"
+    messages: Final = [{"role": "user", "content": "Write a long sentence about a fox. " * 30}]
+    text: Final = "The quick brown fox jumps over the lazy dog. " * 20
+    logging_obj: Final = Logging(
+        model=model,
+        messages=messages,
+        stream=True,
+        call_type="completion",
+        start_time=datetime.datetime(2025, 3, 3, 21, 38, 10),
+        litellm_call_id="cut-no-usage-call",
+        function_id="cut-no-usage-fn",
+    )
+    logging_obj.update_environment_variables(
+        model=model, user=None, optional_params={}, litellm_params={"litellm_call_id": "cut-no-usage-call"}
+    )
+    wrapper: Final = CustomStreamWrapper(
+        completion_stream=ModelResponseListIterator(
+            model_responses=[
+                ModelResponseStream(
+                    id="chatcmpl-cut-no-usage",
+                    created=1741037890,
+                    model=model,
+                    choices=[StreamingChoices(index=0, delta=Delta(role="assistant", content=text))],
+                )
+            ]
+        ),
+        model=model,
+        custom_llm_provider="openai",
+        logging_obj=logging_obj,
+        stream_options=None,
+    )
+
+    with pytest.raises(MidStreamFallbackError):
+        await _collect(wrapper, sync_mode)
+
+    expected: Final = (
+        litellm.token_counter(model=model, messages=messages),
+        litellm.token_counter(model=model, text=text, count_response_tokens=True),
+    )
+    usage: Final = logging_obj.model_call_details["combined_usage_object"]
+    assert (usage.prompt_tokens, usage.completion_tokens) == expected
+    rates: Final = litellm.model_cost[model]
+    spend: Final = expected[0] * rates["input_cost_per_token"] + expected[1] * rates["output_cost_per_token"]
+    assert logging_obj.model_call_details["response_cost"] == pytest.approx(spend)
+
+
 @pytest.mark.parametrize("sync_mode", [True, False])
 @pytest.mark.asyncio
 async def test_stream_with_usage_after_the_finish_chunk_bills_only_the_reported_usage(sync_mode: bool) -> None:
@@ -7953,6 +8251,86 @@ def test_aamazing_unit_test_custom_stream_wrapper_n():
         ), f"idx={idx} translated chunk = {chunk_dict} != openai chunk = {chunks[idx]}"
 
 
+@pytest.mark.parametrize("async_stream", [False, True])
+@pytest.mark.parametrize("transport", ["sdk", "http_handler"])
+@pytest.mark.parametrize("separate_finish", [False, True])
+@pytest.mark.parametrize(
+    "delta_kind,finish_reason,expected",
+    [
+        ("tool", "stop", "tool_calls"),
+        ("tool", "length", "length"),
+        ("tool", "content_filter", "content_filter"),
+        ("tool", "tool_calls", "tool_calls"),
+        ("empty_tools", "stop", "stop"),
+        ("text", "stop", "stop"),
+        ("function", "stop", "stop"),
+    ],
+)
+async def test_openai_compatible_tool_stream_finish_reason(
+    async_stream: bool,
+    transport: str,
+    separate_finish: bool,
+    delta_kind: str,
+    finish_reason: str,
+    expected: str,
+    logging_obj: Logging,
+) -> None:
+    function: Final = {"name": "get_weather", "arguments": "{}"}
+    delta: Final = {
+        "tool": {"tool_calls": [{"index": 0, "id": "call_1", "type": "function", "function": function}]},
+        "empty_tools": {"content": "Hello", "tool_calls": []},
+        "text": {"content": "Hello"},
+        "function": {"function_call": function},
+    }[delta_kind]
+    choices: Final = (
+        {"index": 0, "delta": {"role": "assistant", **delta}, "finish_reason": None if separate_finish else finish_reason},
+        *(({"index": 0, "delta": {}, "finish_reason": finish_reason},) if separate_finish else ()),
+    )
+    body: Final = "".join(
+        "data: " + json.dumps({
+            "id": "chatcmpl-tool-stop", "object": "chat.completion.chunk", "created": 1,
+            "model": "m", "choices": [choice],
+        }) + "\n\n"
+        for choice in choices
+    ) + "data: [DONE]\n\n"
+    raw_response: Final = httpx.Response(
+        200, content=body, headers={"content-type": "text/event-stream"},
+        request=httpx.Request("POST", "https://provider.example/v1/chat/completions"),
+    )
+    client: Final = AsyncOpenAI(api_key="unused") if async_stream else OpenAI(api_key="unused")
+    stream: Final = (
+        (
+            AsyncStream(cast_to=ChatCompletionChunk, response=raw_response, client=client)
+            if isinstance(client, AsyncOpenAI)
+            else Stream(cast_to=ChatCompletionChunk, response=raw_response, client=client)
+        )
+        if transport == "sdk"
+        else OpenAIChatCompletionStreamingHandler(
+            streaming_response=raw_response.aiter_lines() if async_stream else raw_response.iter_lines(),
+            sync_stream=not async_stream,
+        )
+    )
+    response: Final = CustomStreamWrapper(
+        completion_stream=stream, model="m", logging_obj=logging_obj,
+        custom_llm_provider="openai" if transport == "sdk" else "hosted_vllm",
+    )
+    try:
+        received: Final = [chunk async for chunk in response] if async_stream else list(response)
+        assert [chunk.choices[0].finish_reason for chunk in received if chunk.choices[0].finish_reason] == [expected]
+        assembled: Final = litellm.stream_chunk_builder(received)
+        assert assembled.choices[0].finish_reason == expected
+        assert litellm.stream_chunk_builder(response.chunks).choices[0].finish_reason == expected
+        if delta_kind == "tool":
+            assert [tool.model_dump() for tool in assembled.choices[0].message.tool_calls] == [
+                {"id": "call_1", "type": "function", "function": function}
+            ]
+    finally:
+        if isinstance(client, AsyncOpenAI):
+            await client.close()
+        else:
+            client.close()
+
+
 def test_unit_test_custom_stream_wrapper_function_call():
     """
     Test if model returns a tool call, the finish reason is correctly set to 'tool_calls'
@@ -8502,10 +8880,8 @@ async def test_no_synthetic_finish_reason_logged_when_provider_sent_none(sync_mo
         custom_llm_provider="bedrock",
         logging_obj=logging_obj,
     )
-    if sync_mode:
-        list(response)
-    else:
-        [c async for c in response]
+    with pytest.raises(MidStreamFallbackError):
+        await _collect(response, sync_mode)
 
     assert response.received_finish_reason is None
     assert all(not (c.choices and c.choices[0].finish_reason) for c in response.chunks)

tests/unit/llms/a2a/test_common_utils.py

@@ -2,6 +2,7 @@
 
 from collections.abc import Mapping
 from types import MappingProxyType
+from typing import Final
 
 import pytest
 
@@ -50,3 +51,42 @@ async def test_agent_without_entra_credentials_gets_no_bearer():
 
     assert header is None
     assert resolver.calls == []
+
+
+@pytest.mark.parametrize(
+    ("api_key", "expected"),
+    (
+        ("secret", "Bearer secret"),
+        ("Bearer secret", "Bearer Bearer secret"),
+        ("os.environ/SYNTHETIC_KEY", "Bearer os.environ/SYNTHETIC_KEY"),
+        ("", None),
+        (None, None),
+        (42, None),
+    ),
+)
+async def test_plain_agent_key_is_used_verbatim_only_for_nonempty_strings(
+    api_key: object, expected: str | None
+) -> None:
+    header: Final = await resolve_a2a_hop_auth_header({"api_key": api_key}, None)
+    assert header == ({"Authorization": expected} if expected is not None else None)
+
+
+@pytest.mark.parametrize("provider", ("azure_ai", "databricks_agent"))
+async def test_completion_bridge_api_key_is_not_used_for_a2a_hop(provider: str) -> None:
+    assert await resolve_a2a_hop_auth_header({"api_key": "secret"}, provider) is None
+
+
+async def test_entra_bearer_overrides_agent_key_and_passthrough() -> None:
+    resolver: Final = _RecordingEntraResolver()
+    params: Final = {**_SERVICE_PRINCIPAL, "api_key": "secret"}
+    header: Final = await resolve_a2a_hop_auth_header(params, None, resolver, {"authorization": "Bearer forwarded"})
+    assert header == {"Authorization": "Bearer minted-entra-token"}
+    assert resolver.calls == [params]
+
+
+@pytest.mark.parametrize("name", ("Authorization", "aUtHoRiZaTiOn"))
+async def test_passthrough_authorization_takes_precedence_over_plain_agent_key(name: str) -> None:
+    assert (
+        await resolve_a2a_hop_auth_header({"api_key": "secret"}, None, agent_extra_headers={name: "Bearer forwarded"})
+        is None
+    )

tests/unit/llms/azure/test_azure_common_utils.py

@@ -467,6 +467,9 @@ def test_default_max_retries_env_var_reaches_azure_sdk_client():
             "add_message",
             "arun_thread_stream",
             "aresponses",
+            "aget_responses",
+            "adelete_responses",
+            "acancel_responses",
             "aresponses_websocket",
             "alist_input_items",
             "acreate_fine_tuning_job",

tests/unit/llms/bedrock/chat/agentcore/test_agentcore_transformation.py

@@ -19,8 +19,9 @@ from typing import Final
 from unittest.mock import MagicMock, Mock, patch
 
 import litellm
+from litellm.exceptions import MidStreamFallbackError
 from litellm.llms.bedrock.chat.agentcore.transformation import AmazonAgentCoreConfig
-from litellm.llms.custom_httpx.http_handler import HTTPHandler
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 
 
 class TestAgentCoreAcceptHeader:
@@ -1271,6 +1272,87 @@ def test_agentcore_runtime_response_becomes_assistant_message(
     assert response.choices[0].finish_reason == "stop"
 
 
+_AGENTCORE_RUNTIME_URL: Final = (
+    "https://bedrock-agentcore.us-west-2.amazonaws.com/runtimes/"
+    "arn%3Aaws%3Abedrock-agentcore%3Aus-west-2%3A888602223428%3Aruntime%2Fhosted_agent_13sf6-cALnp38iZD/invocations"
+)
+_AGENTCORE_DELTAS: Final = (
+    'data: {"event":{"messageStart":{"role":"assistant"}}}\n\n'
+    'data: {"event":{"contentBlockDelta":{"delta":{"text":"Machine learning lets "},"contentBlockIndex":0}}}\n\n'
+    'data: {"event":{"contentBlockDelta":{"delta":{"text":"computers learn from data."},"contentBlockIndex":0}}}\n\n'
+)
+
+
+async def _stream_agentcore(body: str, sync_mode: bool) -> list[tuple[str | None, str | None]]:
+    """Stream `body` from the runtime as SSE and return each chunk's (content, finish_reason)."""
+    kwargs: Final = {
+        "model": "bedrock/agentcore/arn:aws:bedrock-agentcore:us-west-2:888602223428:runtime/"
+        "hosted_agent_13sf6-cALnp38iZD",
+        "messages": [{"role": "user", "content": "Explain machine learning in simple terms"}],
+        "aws_access_key_id": "AKIAAGENTCOREKEY",
+        "aws_secret_access_key": "agentcore-secret",
+        "stream": True,
+    }
+    if sync_mode:
+        return [
+            (chunk.choices[0].delta.content, chunk.choices[0].finish_reason)
+            for chunk in litellm.completion(**kwargs, client=HTTPHandler())
+        ]
+    return [
+        (chunk.choices[0].delta.content, chunk.choices[0].finish_reason)
+        async for chunk in await litellm.acompletion(
+            **kwargs, client=AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport())
+        )
+    ]
+
+
+@pytest.mark.parametrize(
+    "ending",
+    [
+        pytest.param(
+            'data: {"event":{"contentBlockStop":{"contentBlockIndex":0}}}\n\n'
+            'data: {"event":{"messageStop":{"stopReason":"end_turn"}}}\n\n',
+            id="message_stop",
+        ),
+        pytest.param(
+            'data: {"event":{"messageStop":{"stopReason":"end_turn"}}}\n\n'
+            'data: {"event":{"metadata":{"usage":{"inputTokens":9,"outputTokens":7,"totalTokens":16}}}}\n\n'
+            'data: {"message":{"role":"assistant",'
+            '"content":[{"text":"Machine learning lets computers learn from data."}]}}\n\n',
+            id="strands_full",
+        ),
+    ],
+)
+@pytest.mark.parametrize("sync_mode", [True, False])
+@pytest.mark.respx(assert_all_called=True)
+async def test_agentcore_stream_that_signals_its_end_completes(
+    ending: str, sync_mode: bool, respx_mock: respx.Router
+) -> None:
+    respx_mock.post(_AGENTCORE_RUNTIME_URL).mock(
+        return_value=httpx.Response(
+            200, content=(_AGENTCORE_DELTAS + ending).encode(), headers={"content-type": "text/event-stream"}
+        )
+    )
+
+    chunks: Final = await _stream_agentcore(_AGENTCORE_DELTAS + ending, sync_mode)
+
+    assert "".join(content for content, _ in chunks if content) == "Machine learning lets computers learn from data."
+    assert [finish for _, finish in chunks if finish] == ["stop"]
+
+
+@pytest.mark.parametrize("sync_mode", [True, False])
+@pytest.mark.respx(assert_all_called=True)
+async def test_agentcore_stream_cut_before_it_signals_its_end_raises(sync_mode: bool, respx_mock: respx.Router) -> None:
+    respx_mock.post(_AGENTCORE_RUNTIME_URL).mock(
+        return_value=httpx.Response(
+            200, content=_AGENTCORE_DELTAS.encode(), headers={"content-type": "text/event-stream"}
+        )
+    )
+
+    with pytest.raises(MidStreamFallbackError, match="Stream ended before a finish_reason was received"):
+        await _stream_agentcore(_AGENTCORE_DELTAS, sync_mode)
+
+
 def test_get_complete_url_sends_a_non_string_runtime_endpoint_to_that_endpoint():
     vpc_endpoint: Final = "https://vpce-1.bedrock-agentcore.us-west-2.vpce.amazonaws.com"
 

tests/unit/llms/deepgram/test_deepgram_common_utils.py

@@ -7,6 +7,7 @@ import pytest
 
 import litellm
 from litellm.llms.deepgram.common_utils import (
+    DeepgramSpeakMeter,
     deepgram_listen_addon_pricing_models,
     deepgram_listen_audio_seconds,
     deepgram_listen_callback_params,
@@ -18,7 +19,12 @@ from litellm.llms.deepgram.common_utils import (
     deepgram_listen_requested_model,
     deepgram_listen_transcript,
     deepgram_listen_websocket_target,
+    deepgram_speak_character_rate,
+    deepgram_speak_registry_key,
+    deepgram_speak_requested_model,
+    deepgram_speak_websocket_target,
 )
+from litellm.types.passthrough_endpoints.pass_through_endpoints import WebSocketSessionUsage
 
 NOVA_3_URL: Final = "wss://api.deepgram.com/v1/listen?model=nova-3&encoding=linear16&sample_rate=16000"
 
@@ -324,3 +330,102 @@ def test_deepgram_listen_is_priced(
 )
 def test_deepgram_listen_addon_pricing_models(upstream_url: str, expected: tuple[str, ...]):
     assert deepgram_listen_addon_pricing_models(upstream_url) == expected
+
+
+@pytest.mark.parametrize(
+    ("api_base", "query_string", "expected"),
+    [
+        pytest.param(
+            None,
+            "encoding=linear16",
+            "wss://api.deepgram.com/v1/speak?encoding=linear16&model=aura-asteria-en",
+            id="no model: Deepgram's own default, named so billing and upstream agree",
+        ),
+        pytest.param(
+            None,
+            "model=aura-asteria-en&model=aura-2-thalia-en",
+            "wss://api.deepgram.com/v1/speak?model=aura-asteria-en",
+            id="second model dropped: Deepgram honours the last, auth and pricing read the first",
+        ),
+        pytest.param(
+            "http://localhost:8080/v1",
+            "model=aura-2-thalia-en",
+            "ws://localhost:8080/v1/speak?model=aura-2-thalia-en",
+            id="server-configured api base",
+        ),
+    ],
+)
+def test_deepgram_speak_websocket_target(api_base: str | None, query_string: str, expected: str):
+    assert deepgram_speak_websocket_target(api_base, query_string) == expected
+
+
+@pytest.mark.parametrize(
+    "query_string",
+    [
+        "",
+        "model=",
+        "model=&model=aura-2-thalia-en",
+        "model=aura-2-thalia-en",
+        "model=aura-asteria-en&model=aura-2-thalia-en",
+        "encoding=mulaw",
+    ],
+)
+def test_deepgram_speak_requested_voice_is_the_only_voice_the_upstream_target_carries(query_string: str):
+    """Authorization runs against the requested voice and the upstream URL is built separately, so they must agree or
+    a key could be authorized for one voice and reach another"""
+    target: Final = deepgram_speak_websocket_target(None, query_string)
+    assert parse_qs(urlparse(target).query)["model"] == [deepgram_speak_requested_model(query_string)]
+
+
+@pytest.mark.usefixtures("local_model_cost_map")
+@pytest.mark.parametrize(
+    ("model", "registry_key", "priced"),
+    [
+        ("aura-asteria-en", "deepgram/aura", True),
+        ("aura-2-thalia-en", "deepgram/aura-2", True),
+        ("aura-2-celeste-es", "deepgram/aura-2", True),
+        ("aura-99-x-en", "deepgram/aura-99", False),
+        ("aura-2", "deepgram/aura-2", True),
+        ("nova-3", "deepgram/nova-3", False),
+    ],
+)
+def test_deepgram_speak_prices_each_voice_at_its_generation_row_only(model: str, registry_key: str, priced: bool):
+    """A voice is never billed at another generation's rate, and a listen model's row is not a speak price"""
+    upstream_url: Final = f"wss://api.deepgram.com/v1/speak?model={model}"
+
+    assert deepgram_speak_registry_key(upstream_url) == registry_key
+    assert deepgram_speak_character_rate(upstream_url) == (
+        litellm.model_cost[registry_key]["input_cost_per_character"] if priced else None
+    )
+
+
+@pytest.mark.parametrize(
+    ("frame", "characters"),
+    [
+        ('{"type": "Speak", "text": "Hello world."}', 12),
+        ('{"type":"Speak","text":"¿qué tal?"}', 9),
+        ('{"type": "Flush"}', 0),
+        ('{"type": "Speak"}', 0),
+        ('{"type": "Speak", "text": 12}', 0),
+        ('["Speak", "Hello"]', 0),
+        ("Hello", 0),
+        ("[" * 100_000, 0),
+    ],
+)
+def test_deepgram_speak_meter_measures_only_speak_text_and_never_raises(frame: str, characters: int):
+    meter: Final = DeepgramSpeakMeter("wss://api.deepgram.com/v1/speak?model=aura-2-thalia-en", character_rate=1.0)
+
+    assert meter.measure(frame) == characters
+
+
+@pytest.mark.parametrize(
+    ("upstream_url", "model"),
+    [
+        ("wss://api.deepgram.com/v1/speak?model=aura-2-thalia-en", "aura-2-thalia-en"),
+        ("wss://api.deepgram.com/v1/speak?encoding=linear16&model=aura-asteria-en", "aura-asteria-en"),
+    ],
+)
+def test_deepgram_speak_meter_prices_the_total_at_the_rate_it_holds(upstream_url: str, model: str) -> None:
+    assert DeepgramSpeakMeter(upstream_url, character_rate=0.25).price(47) == WebSocketSessionUsage(
+        model=model, custom_llm_provider="deepgram", response_cost=47 * 0.25
+    )

tests/unit/llms/langgraph/chat/test_langgraph_transformation.py

@@ -22,12 +22,14 @@ import asyncio
 import importlib
 import json
 import os
+from collections.abc import AsyncIterator
+from typing import Final
 
 import httpx
 import pytest
 
 import litellm
-from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
 from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome
 
 
@@ -268,3 +270,28 @@ async def test_langgraph_streaming_acompletion_without_a_client_posts_through_th
         "assistant_id": "agent",
         "input": {"messages": [{"role": "human", "content": "What is 25 * 4?"}]},
     }
+
+
+@pytest.mark.asyncio
+async def test_langgraph_async_stream_delivers_every_step_of_the_run_before_its_stop() -> None:
+    async def events() -> AsyncIterator[bytes]:
+        question: Final = {"type": "human", "content": "What is 25 * 4?"}
+        answers: Final = ({"type": "ai", "content": "25 * 4"}, {"type": "ai", "content": " is 100"})
+        yield b'event: metadata\ndata: {"run_id": "run-1", "attempt": 1}\n\n'
+        for step in range(len(answers) + 1):
+            yield b"event: values\ndata: " + json.dumps({"messages": [question, *answers[:step]]}).encode() + b"\n\n"
+
+    client: Final = AsyncHTTPHandler(
+        transport=httpx.MockTransport(lambda request: httpx.Response(200, content=events()))
+    )
+
+    response: Final = await litellm.acompletion(
+        model="langgraph/agent",
+        messages=[{"role": "user", "content": "What is 25 * 4?"}],
+        api_base="http://langgraph.test",
+        stream=True,
+        client=client,
+    )
+    streamed: Final = [(chunk.choices[0].delta.content, chunk.choices[0].finish_reason) async for chunk in response]
+
+    assert streamed == [("25 * 4", None), (" is 100", None), (None, "stop")]

tests/unit/llms/oci/chat/test_oci_chat_transformation.py

@@ -1,4 +1,5 @@
 import datetime
+import copy
 import logging
 from typing import Final
 import httpx
@@ -16,6 +17,8 @@ from litellm.llms.oci.chat.transformation import (
     OCIRequestWrapper,
     version,
 )
+from litellm.types.llms.oci import OCIChatRequestPayload, OCICompletionPayload
+from litellm.types.llms.openai import AllMessageValues
 
 TEST_MODEL_NAME = "xai.grok-4"
 TEST_MODEL = f"oci/{TEST_MODEL_NAME}"
@@ -2205,3 +2208,74 @@ def test_transform_request_sends_a_pdf_file_part_as_an_image_data_url() -> None:
             ],
         }
     ]
+
+
+@pytest.mark.parametrize("model", ["google.gemini-2.5-flash", "oci/google.gemini-2.5-flash"])
+@pytest.mark.parametrize("multipart", [False, True])
+@pytest.mark.parametrize("interleaved", [False, True])
+def test_gemini_preserves_all_system_instructions(model: str, multipart: bool, interleaved: bool) -> None:
+    first: Final = "End every answer with PINEAPPLE."
+    second: Final = "Start every answer with BANANA."
+    conversation: Final[list[AllMessageValues]] = [
+        {"role": "user", "content": "What is 2+2?"},
+        {"role": "assistant", "content": "Four."},
+    ]
+    messages: Final[list[AllMessageValues]] = [
+        {"role": "system", "content": first},
+        *(conversation if interleaved else []),
+        {"role": "system", "content": [{"type": "text", "text": second}] if multipart else second},
+        *(conversation if not interleaved else []),
+        {"role": "user", "content": "Repeat it."},
+    ]
+    original: Final = copy.deepcopy(messages)
+    payload: Final = OCICompletionPayload.model_validate(
+        OCIChatConfig().transform_request(
+            model=model,
+            messages=messages,
+            optional_params={"oci_compartment_id": TEST_COMPARTMENT_ID},
+            litellm_params={},
+            headers={},
+        )
+    )
+    chat: Final = payload.chatRequest
+    assert isinstance(chat, OCIChatRequestPayload)
+    assert [message.role for message in chat.messages] == ["SYSTEM", "USER", "ASSISTANT", "USER"]
+    assert chat.messages[0].model_dump(exclude_none=True) == {
+        "role": "SYSTEM",
+        "content": (
+            [{"type": "TEXT", "text": first}, {"type": "TEXT", "text": second}]
+            if multipart
+            else [{"type": "TEXT", "text": f"{first}\n\n{second}"}]
+        ),
+    }
+    assert [message.model_dump(exclude_none=True) for message in chat.messages[1:]] == [
+        {"role": "USER", "content": [{"type": "TEXT", "text": "What is 2+2?"}]},
+        {"role": "ASSISTANT", "content": [{"type": "TEXT", "text": "Four."}]},
+        {"role": "USER", "content": [{"type": "TEXT", "text": "Repeat it."}]},
+    ]
+    assert messages == original
+
+
+@pytest.mark.parametrize("model", ["xai.grok-4", "meta.llama-4-maverick"])
+def test_other_oci_models_preserve_system_message_positions(model: str) -> None:
+    messages: Final[list[AllMessageValues]] = [
+        {"role": "system", "content": "Agent prompt."},
+        {"role": "user", "content": "Hello."},
+        {"role": "system", "content": "Conversation summary."},
+    ]
+    payload: Final = OCICompletionPayload.model_validate(
+        OCIChatConfig().transform_request(
+            model=model,
+            messages=messages,
+            optional_params={"oci_compartment_id": TEST_COMPARTMENT_ID},
+            litellm_params={},
+            headers={},
+        )
+    )
+    chat: Final = payload.chatRequest
+    assert isinstance(chat, OCIChatRequestPayload)
+    assert [message.model_dump(exclude_none=True) for message in chat.messages] == [
+        {"role": "SYSTEM", "content": [{"type": "TEXT", "text": "Agent prompt."}]},
+        {"role": "USER", "content": [{"type": "TEXT", "text": "Hello."}]},
+        {"role": "SYSTEM", "content": [{"type": "TEXT", "text": "Conversation summary."}]},
+    ]

tests/unit/llms/openai/responses/test_openai_responses_guardrail_handler.py

@@ -29,6 +29,7 @@ from openai.types.responses import (
     ResponseOutputRefusal,
     ResponseOutputText,
 )
+from openai.types.responses.response_compaction_item import ResponseCompactionItem
 
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@@ -3349,6 +3350,25 @@ class TestGetStructuredMessages:
         result = handler.get_structured_messages(data)
         assert result is None
 
+    def test_should_return_none_for_a_moderation_style_list_of_strings(self):
+        handler = OpenAIResponsesHandler()
+        data = {"model": "omni-moderation-latest", "input": ["safe text", "more safe text"]}
+        assert handler.get_structured_messages(data) is None
+
+    def test_should_keep_an_image_item_replayed_with_an_sdk_compaction_item(self):
+        handler = OpenAIResponsesHandler()
+        image_url = "https://example.com/cat.png"
+        data = {
+            "model": "gpt-5.5",
+            "input": [
+                ResponseCompactionItem(type="compaction", id="cmp_1", encrypted_content="abc"),
+                {"role": "user", "content": [{"type": "input_image", "image_url": image_url}]},
+            ],
+        }
+        result = handler.get_structured_messages(data)
+        assert result is not None
+        assert image_url in json.dumps(result)
+
 
 class ToolAppendingGuardrail(CustomGuardrail):
     """Guardrail that appends a new function tool, mimicking a guardrail that

tests/unit/llms/openai_like/test_kyma_provider.py

@@ -410,8 +410,12 @@ def test_cost_headers_show_the_discount_and_margin_taken_from_the_cost_kyma_repo
     assert float(headers["x-litellm-response-cost-discount-amount"] or "nan") == pytest.approx(0.004)
     assert float(headers["x-litellm-response-cost-margin-amount"] or "nan") == pytest.approx(0.0004)
     assert float(headers["x-litellm-response-cost"] or "nan") == pytest.approx(0.0044)
-    assert float(headers["x-litellm-response-cost-input"] or "nan") == 0.0
-    assert float(headers["x-litellm-response-cost-output"] or "nan") == pytest.approx(0.0044)
+    row: Final = litellm.get_model_info("sonar", custom_llm_provider="kyma")
+    priced_input: Final = 4 * row["input_cost_per_token"]
+    priced_output: Final = 3 * row["output_cost_per_token"]
+    input_share: Final = priced_input / (priced_input + priced_output)
+    assert float(headers["x-litellm-response-cost-input"] or "nan") == pytest.approx(0.0044 * input_share)
+    assert float(headers["x-litellm-response-cost-output"] or "nan") == pytest.approx(0.0044 * (1 - input_share))
 
 
 @pytest.mark.parametrize("thinking", ({"type": "enabled", "budget_tokens": 1024}, {"type": "adaptive"}))

tests/unit/proxy/agent_endpoints/test_a2a_endpoints.py

@@ -18,6 +18,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import httpx
 import pytest
 import respx
+from fastapi.testclient import TestClient
 from pydantic import BaseModel, ConfigDict
 
 import litellm
@@ -2941,3 +2942,56 @@ async def test_failed_upstream_stream_relays_its_body_only_when_that_is_a_json_o
         "id": "req-1",
         "method": "tasks/resubscribe",
     }
+
+
+def _a2a_app_refusing_with(refusal: Exception) -> TestClient:
+    from fastapi import FastAPI
+
+    from litellm.proxy.agent_endpoints.a2a_endpoints import router
+    from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+    def refuse() -> UserAPIKeyAuth:
+        raise refusal
+
+    app: Final = FastAPI()
+    app.include_router(router)
+    app.dependency_overrides[user_api_key_auth] = refuse
+    return TestClient(app)
+
+
+@pytest.mark.parametrize("path", ["/a2a/agent-1", "/a2a/agent-1/message/send", "/v1/a2a/agent-1/message/send"])
+def test_a2a_auth_refusals_answer_as_jsonrpc_errors_with_the_callers_id(path: str):
+    from litellm.proxy._types import ProxyException
+
+    client: Final = _a2a_app_refusing_with(
+        ProxyException(message="Budget has been exceeded!", type="budget_exceeded", param=None, code=422)
+    )
+
+    response: Final = client.post(path, json={"jsonrpc": "2.0", "id": "req-7", "method": "message/send"})
+
+    assert response.status_code == 422
+    assert response.json() == {
+        "jsonrpc": "2.0",
+        "id": "req-7",
+        "error": {
+            "code": -32000,
+            "message": "Budget has been exceeded!",
+            "data": {"type": "budget_exceeded", "param": None, "code": "422"},
+        },
+    }
+
+
+def test_a2a_http_refusals_answer_as_jsonrpc_errors_with_a_null_id_for_unparseable_bodies():
+    from fastapi import HTTPException
+
+    client: Final = _a2a_app_refusing_with(HTTPException(status_code=403, detail="Agent 'agent-1' is not allowed"))
+
+    response: Final = client.post("/a2a/agent-1", content=b"not json")
+
+    assert response.status_code == 403
+    body: Final = response.json()
+    assert body["jsonrpc"] == "2.0"
+    assert body["id"] is None
+    assert body["error"]["code"] == -32000
+    assert body["error"]["message"] == "Agent 'agent-1' is not allowed"
+    assert body["error"]["data"]["code"] == "403"