Merge remote-tracking branch 'origin/main' into litellm_feat_d3d075

litellm-proxy-extras/litellm_proxy_extras/migrations/20261016000000_background_interaction_settlement_settled_at_index/migration.sql

@@ -0,0 +1,2 @@
+CREATE INDEX CONCURRENTLY IF NOT EXISTS "idx_background_interaction_settlement_settled_at"
+ON "LiteLLM_BackgroundInteractionSettlement" ("settled_at");

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -1981,6 +1981,7 @@ model LiteLLM_BackgroundInteractionSettlement {
   outcome             String?
 
   @@index([claimed_at], map: "idx_background_interaction_settlement_claimed_at")
+  @@index([settled_at], map: "idx_background_interaction_settlement_settled_at")
 }
 
 model LiteLLM_Lens {

litellm/__init__.py

@@ -207,7 +207,7 @@ _async_failure_callback: List[Union[str, Callable[..., object], "CustomLogger"]]
     []
 )  # internal variable - async custom callbacks are routed here.
 pre_call_rules: List[Callable] = []
-post_call_rules: List[Callable] = []
+post_call_rules: List[Callable[..., object]] = []
 turn_off_message_logging: Optional[bool] = False
 standard_logging_payload_excluded_fields: Optional[List[str]] = (
     None  # Fields to exclude from StandardLoggingPayload before callbacks receive it
@@ -393,6 +393,7 @@ key_alias_pattern: str | None = None
 enable_gemini_default_thinking_level_low: bool = (
     False  # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping
 )
+gemini_include_thoughts_default: bool = True
 ####################
 logging: bool = True
 enable_loadbalancing_on_batch_endpoints: Optional[bool] = None

litellm/caching/caching.py

@@ -25,7 +25,7 @@ from litellm._internal_context import current_service_target, service_target
 from litellm._logging import verbose_logger
 from litellm.constants import CACHED_STREAMING_CHUNK_DELAY
 from litellm.integrations.otel.runtime import phase_span
-from litellm.litellm_core_utils.model_param_helper import ModelParamHelper
+from litellm.litellm_core_utils.model_param_helper import ModelParamHelper, peek_reasoning_summary_aliases
 from litellm.types.caching import *
 from litellm.types.utils import EmbeddingResponse, is_litellm_owned_kwarg
 
@@ -376,7 +376,7 @@ class Cache:
         )
         return "".join(f"{field}: {value}" for field, value in scope_values if value is not None)
 
-    def get_cache_key(self, **kwargs) -> str:
+    def get_cache_key(self, **kwargs: object) -> str:
         """
         Get the cache key for the given arguments.
 
@@ -410,13 +410,16 @@ class Cache:
                 if litellm.enable_caching_on_provider_specific_optional_params is True:  # feature flagged for now
                     if kwargs[param] is None:
                         continue  # ignore None params
-                    param_value = kwargs[param]
-                    cache_key += f"{param}: {param_value}"
+                    cache_key += f"{param}: {kwargs[param]}"
 
         if is_semantic_cache:
             cache_key += self._get_semantic_cache_tenant_scope(kwargs)
 
-        hashed_cache_key = Cache._get_hashed_cache_key(cache_key)
+        reasoning_summary: Final = peek_reasoning_summary_aliases(kwargs)
+        reasoning_summary_material: Final = (
+            "" if reasoning_summary is None else f"reasoning_summary: {reasoning_summary}"
+        )
+        hashed_cache_key = Cache._get_hashed_cache_key(cache_key + reasoning_summary_material)
         hashed_cache_key = self._add_namespace_to_cache_key(hashed_cache_key, **kwargs)
         verbose_logger.debug(
             "\nCreated cache key: %s (source material length: %d)",

litellm/constants.py

@@ -57,6 +57,7 @@ RUNTIME_UPDATABLE_ROUTER_SETTINGS: Final[frozenset[str]] = frozenset(
         "model_group_retry_policy",
         "model_group_alias",
         "enable_weighted_failover",
+        "enable_mid_stream_fallback_continuation",
         "enable_tag_filtering",
         "tag_routing_prefix",
         "optional_pre_call_checks",
@@ -146,6 +147,7 @@ HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
 # Streams exceeding this duration are terminated with a Timeout error.
 # None (default) = no limit.  Set env var to a number of seconds to enable globally.
 LITELLM_MAX_STREAMING_DURATION_SECONDS: Final = get_env_float_or_none("LITELLM_MAX_STREAMING_DURATION_SECONDS")
+MID_STREAM_FAILURE_LOG_WAIT_SECONDS: Final = 1.0
 
 # Maximum number of base64 characters to keep in logging payloads.
 # Data URIs exceeding this are replaced with a size placeholder.
@@ -1629,6 +1631,7 @@ INTERNAL_CALL_ORIGIN_METADATA_KEY: Final = "internal_call_origin"
 MANAGED_BACKGROUND_RESPONSE_METADATA_KEY: Final = "_managed_background_response"
 SESSION_ID_GENERATED_METADATA_KEY: Final = "litellm_session_id_generated"
 SESSION_ID_OMITTED_METADATA_KEY: Final = "litellm_session_id_omitted"
+CALLER_SESSION_ID_METADATA_KEY: Final = "litellm_caller_session_id"
 LITELLM_TRUNCATED_PAYLOAD_FIELD: Final = "litellm_truncated"
 LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE: Final = (
     "Truncation is a DB storage safeguard. "
@@ -1895,6 +1898,7 @@ _background_interaction_cost_polling_env: Final = os.getenv(
     "BACKGROUND_INTERACTION_COST_POLLING_ENABLED", "true"
 ).lower()
 BACKGROUND_INTERACTION_COST_POLLING_ENABLED: Final = _background_interaction_cost_polling_env == "true"
+BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS: Final = 300.0
 PROXY_BUDGET_RESCHEDULER_MAX_TIME: Final = get_env_int("PROXY_BUDGET_RESCHEDULER_MAX_TIME", 605)
 PROXY_BATCH_WRITE_AT: Final = get_env_int("PROXY_BATCH_WRITE_AT", 10)  # in seconds, increased from 10
 PROXY_CONFIG_RELOAD_INTERVAL_SECONDS: Final = get_env_int("PROXY_CONFIG_RELOAD_INTERVAL_SECONDS", 30)

litellm/exceptions.py

@@ -1217,6 +1217,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
         num_retries: int | None = None,
         generated_content: str = "",
         is_pre_first_chunk: bool = False,
+        emitted_disqualifying_content: bool = False,
     ):
         original_status: Final = getattr(original_exception, "status_code", None)
         self.status_code = int(original_status) if original_status is not None else 503
@@ -1229,6 +1230,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
         self.num_retries = num_retries
         self.generated_content = generated_content
         self.is_pre_first_chunk = is_pre_first_chunk
+        self.emitted_disqualifying_content = emitted_disqualifying_content
 
         # Create a response if one wasn't provided
         if response is None:

litellm/integrations/custom_guardrail.py

@@ -695,9 +695,7 @@ class CustomGuardrail(CustomLogger):
         supported_event_hooks: list[GuardrailEventHooks],
     ) -> None:
         allowed_hooks: Final = frozenset(supported_event_hooks) | (
-            frozenset((GuardrailEventHooks.logging_only,))
-            if self.uses_apply_guardrail_interface() and not self.use_native_lifecycle_hooks
-            else frozenset()
+            frozenset((GuardrailEventHooks.logging_only,)) if self.accepts_logging_only_mode() else frozenset()
         )
 
         def _validate_event_hook_list_is_in_supported_event_hooks(
@@ -869,8 +867,13 @@ class CustomGuardrail(CustomLogger):
                     return True
         return False
 
-    def uses_apply_guardrail_interface(self) -> bool:
-        return type(self).apply_guardrail is not CustomGuardrail.apply_guardrail
+    @classmethod
+    def uses_apply_guardrail_interface(cls) -> bool:
+        return cls.apply_guardrail is not CustomGuardrail.apply_guardrail
+
+    @classmethod
+    def accepts_logging_only_mode(cls) -> bool:
+        return cls.uses_apply_guardrail_interface() and not cls.use_native_lifecycle_hooks
 
     @classmethod
     def supports_logging_only_scope(cls) -> bool:
@@ -1029,7 +1032,7 @@ class CustomGuardrail(CustomLogger):
             )
             return kwargs, result
         litellm_params: Final = kwargs.get("litellm_params") or {}
-        scratch_metadata: Final = {
+        scratch_metadata: Final[dict[str, object]] = {  # mutable-ok: apply_guardrail records its verdict here
             key: value
             for key, value in (litellm_params.get("metadata") or {}).items()
             if key != "standard_logging_guardrail_information"
@@ -1083,11 +1086,11 @@ class CustomGuardrail(CustomLogger):
 
     async def _scan_logged_call(
         self,
-        kwargs: dict,  # mutable-ok: CustomLogger.async_logging_hook contract
+        kwargs: dict[str, object],  # mutable-ok: CustomLogger.async_logging_hook contract
         response: object | None,
         translation: "BaseTranslation",
         output_translation: "BaseTranslation",
-        scratch_metadata: dict,  # mutable-ok: apply_guardrail records its verdict into request metadata
+        scratch_metadata: dict[str, object],  # mutable-ok: apply_guardrail records its verdict into request metadata
     ) -> None:
         scratch_fields: Final = self._copy_scratch_request_fields(kwargs)
         scratch_input, scratch_tools = scratch_fields or (None, None)

litellm/integrations/gitlab/gitlab_prompt_manager.py

@@ -219,7 +219,7 @@ class GitLabTemplateManager:
                 elif value.replace(".", "").isdigit():
                     try:
                         result[key] = float(value)
-                    except Exception:
+                    except ValueError:
                         result[key] = value
                 else:
                     result[key] = value.strip("\"'")

litellm/interactions/background_cost_polling.py

@@ -32,19 +32,24 @@ task to settle. A ``BackgroundSettlementStore`` makes the pending settlement
 durable across processes: the create registers the request context that
 billing needs (never provider credentials), the settlement is claimed
 exactly once through the store, and a delete on any replica rebuilds the
-billing context from the store when the poll task is not local. Rows left
-unclaimed by a process that died are resumed at startup. The default store is
-in-memory, which keeps the SDK and single-process behavior unchanged; the
-proxy installs a database-backed one.
+billing context from the store when the poll task is not local. Every row
+still unsettled when a process starts is resumed there, including one whose
+settler died between its claim and its outcome: the database-backed store
+treats a claim as a lease that its settler renews until it records the
+outcome, so such a row is claimable again once the lease runs out. The
+default store is in-memory, which keeps the SDK and single-process behavior
+unchanged; the proxy installs a database-backed one.
 """
 
 import asyncio
+import itertools
 from collections.abc import Awaitable, Callable, Iterable, Iterator, Mapping, Sequence
 from dataclasses import dataclass, field
 from datetime import datetime, timezone
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeAlias
 
+from openai import APIStatusError
 from pydantic import ConfigDict, JsonValue, TypeAdapter, ValidationError
 from pydantic_core import PydanticSerializationError, to_jsonable_python
 
@@ -54,10 +59,12 @@ from litellm.constants import (
     BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS,
     BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS,
     BACKGROUND_INTERACTION_COST_POLLING_ENABLED,
+    BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS,
 )
 from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
 from litellm.types.interactions import InteractionsAPIResponse
 from litellm.types.llms.base import LiteLLMBaseModel
+from litellm.types.router import CredentialLiteLLMParams
 from litellm.types.utils import CustomPricingLiteLLMParams
 
 if TYPE_CHECKING:
@@ -101,6 +108,7 @@ class PendingBackgroundInteraction:
     custom_llm_provider: str
     create_context: BackgroundInteractionCreateContext
     created_at: datetime
+    claim_expires_at: datetime | None = None
 
 
 class BackgroundSettlementStore(Protocol):
@@ -112,9 +120,11 @@ class BackgroundSettlementStore(Protocol):
 
     async def claim(self, interaction_id: str) -> bool: ...
 
+    async def renew_claim(self, interaction_id: str) -> None: ...
+
     async def record_outcome(self, interaction_id: str, outcome: SettlementOutcome) -> None: ...
 
-    async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]: ...
+    async def unsettled(self) -> Sequence[PendingBackgroundInteraction]: ...
 
 
 @dataclass(frozen=True, slots=True)
@@ -145,10 +155,13 @@ class InMemoryBackgroundSettlementStore:
         self._rows[interaction_id] = None
         return True
 
+    async def renew_claim(self, interaction_id: str) -> None:
+        return None
+
     async def record_outcome(self, interaction_id: str, outcome: SettlementOutcome) -> None:
         return None
 
-    async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]:
+    async def unsettled(self) -> Sequence[PendingBackgroundInteraction]:
         return tuple(row for row in self._rows.values() if row is not None)
 
 
@@ -169,11 +182,15 @@ class BackgroundInteractionPollContext:
     interaction_id: str
     custom_llm_provider: str
     logging_obj: "LiteLLMLoggingObj"
-    api_key: str | None = None
-    api_base: str | None = None
+    credentials: CredentialLiteLLMParams = field(default_factory=CredentialLiteLLMParams)
+    deployment_id: str | None = None
+    model_group: str | None = None
     initial_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_INITIAL_INTERVAL_SECONDS
     max_interval_seconds: float = BACKGROUND_INTERACTION_COST_POLL_MAX_INTERVAL_SECONDS
     timeout_seconds: float = BACKGROUND_INTERACTION_COST_POLL_TIMEOUT_SECONDS
+    start_delay_seconds: float = 0.0
+    outcome_write_timeout_seconds: float = BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS / 2
+    claim_renewal_interval_seconds: float = BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS / 3
     store: BackgroundSettlementStore = field(default_factory=InMemoryBackgroundSettlementStore)
     resumed: bool = False
 
@@ -184,15 +201,41 @@ FetchInteraction: TypeAlias = Callable[[BackgroundInteractionPollContext], Await
 async def fetch_background_interaction(context: BackgroundInteractionPollContext) -> InteractionsAPIResponse:
     from litellm.interactions import aget
 
+    credentials: Final = context.credentials
     return await aget(
         interaction_id=context.interaction_id,
         custom_llm_provider=context.custom_llm_provider,
-        api_key=context.api_key,
-        api_base=context.api_base,
+        api_key=credentials.api_key,
+        api_base=credentials.api_base,
+        vertex_project=credentials.vertex_project,
+        vertex_location=credentials.vertex_location,
+        vertex_credentials=credentials.vertex_credentials,
         **{"no-log": True},
     )
 
 
+_FETCH_CREDENTIAL_FIELDS: Final = ("api_key", "api_base", "vertex_project", "vertex_location", "vertex_credentials")
+
+
+def _fetch_credentials(request_kwargs: Mapping[str, object]) -> CredentialLiteLLMParams:
+    try:
+        return CredentialLiteLLMParams.model_validate(
+            {field_name: request_kwargs.get(field_name) for field_name in _FETCH_CREDENTIAL_FIELDS}
+        )
+    except ValidationError:
+        verbose_logger.debug(
+            "Fetching a background interaction with environment credentials: the request's are malformed"
+        )
+        return CredentialLiteLLMParams()
+
+
+_PERMANENT_FETCH_STATUS_CODES: Final = frozenset({400, 401, 403, 404})
+
+
+def _is_permanent_fetch_error(error: Exception) -> bool:
+    return isinstance(error, APIStatusError) and error.status_code in _PERMANENT_FETCH_STATUS_CODES
+
+
 def _poll_intervals(initial: float, maximum: float, timeout: float) -> Iterator[float]:
     elapsed = 0.0  # rebind-ok: the schedule accumulates the time it has already yielded
     interval = initial  # rebind-ok: the schedule doubles the interval up to the cap
@@ -231,6 +274,7 @@ _UNCARRIED_METADATA_KEY: Final = "user_api_key_auth"
 _JSON_VALUE: Final = TypeAdapter(JsonValue)
 _STRING: Final = TypeAdapter(str)
 _OBJECT_MAPPING: Final = TypeAdapter(Mapping[str, object])
+_OBJECT_DICT: Final = TypeAdapter(dict[str, object])
 
 
 def _carries(key: str) -> bool:
@@ -256,10 +300,36 @@ def _as_datetime(start_time: datetime | float) -> datetime:
     return start_time if isinstance(start_time, datetime) else datetime.fromtimestamp(start_time, tz=timezone.utc)
 
 
+def _spend_metadata(logging_obj: "LiteLLMLoggingObj") -> Mapping[str, object]:
+    """
+    The metadata the creating worker's own spend row is built from, with the
+    request tags computed once, header-derived ones included, because a
+    rebuilt logging object has no request headers to derive them from.
+    """
+    litellm_params: Final = _OBJECT_DICT.validate_python(logging_obj.model_call_details.get("litellm_params") or {})
+    return {
+        **_OBJECT_MAPPING.validate_python(get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)),
+        **_request_tags(litellm_params),
+    }
+
+
+def _request_tags(litellm_params: Mapping[str, object]) -> Mapping[str, object]:
+    from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
+
+    try:
+        return {
+            "tags": StandardLoggingPayloadSetup.get_request_tags(
+                litellm_params=_OBJECT_DICT.validate_python(litellm_params),
+                proxy_server_request=_OBJECT_DICT.validate_python(litellm_params.get("proxy_server_request") or {}),
+            )
+        }
+    except AttributeError:
+        verbose_logger.debug("Keeping a background interaction's request tags as sent: they are not a list")
+        return {}
+
+
 def _create_context(logging_obj: "LiteLLMLoggingObj", custom_llm_provider: str) -> BackgroundInteractionCreateContext:
-    metadata: Final = _OBJECT_MAPPING.validate_python(
-        get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
-    )
+    metadata: Final = _spend_metadata(logging_obj)
     litellm_params: Final = _OBJECT_MAPPING.validate_python(logging_obj.litellm_params)
     model: Final = logging_obj.model_call_details.get("model")
     return BackgroundInteractionCreateContext(
@@ -313,6 +383,16 @@ async def _settled_elsewhere(context: BackgroundInteractionPollContext) -> bool:
         return False
 
 
+async def _settled_elsewhere_before_the_window_ends(
+    context: BackgroundInteractionPollContext, intervals: Iterator[float]
+) -> bool:
+    for interval in intervals:
+        await asyncio.sleep(interval)
+        if await _settled_elsewhere(context):
+            return True
+    return False
+
+
 async def _claim(context: BackgroundInteractionPollContext) -> bool | None:
     """
     Exactly-once gate between every settler of one interaction, on every
@@ -326,30 +406,118 @@ async def _claim(context: BackgroundInteractionPollContext) -> bool | None:
         return None
 
 
-async def _record(context: BackgroundInteractionPollContext, outcome: SettlementOutcome) -> SettlementOutcome:
+async def _wrote_outcome(context: BackgroundInteractionPollContext, outcome: SettlementOutcome) -> bool:
     try:
         await context.store.record_outcome(context.interaction_id, outcome)
-    except Exception:  # noqa: BLE001  # the outcome is an audit trail; the claim already made the settlement exclusive
-        verbose_logger.exception("Could not record the settlement of background interaction %s", context.interaction_id)
+    except Exception as e:  # noqa: BLE001  # a failed write is retried while the claim still shuts out takeovers
+        verbose_logger.warning(
+            "Could not record the settlement of background interaction %s: %s", context.interaction_id, e
+        )
+        return False
+    return True
+
+
+async def _recorded(context: BackgroundInteractionPollContext, outcome: SettlementOutcome) -> bool:
+    """
+    Retries a failed write for half the claim lease, while no other settler
+    can take the row over, so a database blip does not leave the row to a
+    takeover or hold the budget reservation that only a recorded release frees.
+    """
+    retries: Final = _poll_intervals(
+        initial=context.initial_interval_seconds,
+        maximum=context.max_interval_seconds,
+        timeout=context.outcome_write_timeout_seconds,
+    )
+    for delay in itertools.chain((0.0,), retries):
+        await asyncio.sleep(delay)
+        if await _wrote_outcome(context, outcome):
+            return True
+    verbose_logger.error(
+        "Could not record the settlement of background interaction %s within %ss; the next process to take "
+        "over its lapsed claim settles it",
+        context.interaction_id,
+        context.outcome_write_timeout_seconds,
+    )
+    return False
+
+
+async def _bill(context: BackgroundInteractionPollContext, response: InteractionsAPIResponse) -> SettlementOutcome:
+    await _bill_settled_interaction(logging_obj=context.logging_obj, response=response)
+    await _recorded(context, "billed")
+    return "billed"
+
+
+async def _release(
+    context: BackgroundInteractionPollContext, outcome: Literal["released", "unsettled"]
+) -> SettlementOutcome:
+    """
+    A release writes no spend row, so a takeover cannot tell whether the
+    settler before it released. The outcome is recorded first and the
+    reservation released only once it is, which leaves a takeover with
+    nothing to release twice.
+    """
+    if await _recorded(context, outcome):
+        await _release_open_budget_reservation(logging_obj=context.logging_obj)
     return outcome
 
 
+async def _renew_claim_until_cancelled(context: BackgroundInteractionPollContext) -> None:
+    while True:
+        await asyncio.sleep(context.claim_renewal_interval_seconds)
+        try:
+            await context.store.renew_claim(context.interaction_id)
+        except Exception as e:  # noqa: BLE001  # a missed renewal is retried next interval, well inside the lease
+            verbose_logger.warning(
+                "Could not renew the settlement claim of background interaction %s: %s", context.interaction_id, e
+            )
+
+
+async def _settle(
+    context: BackgroundInteractionPollContext, response: InteractionsAPIResponse | None
+) -> SettlementOutcome:
+    """
+    Billing runs every success callback in turn, and a slow one can outlast
+    the claim lease, so the claim is renewed until the outcome is recorded.
+    A settler that stops renewing has died, and only then can another take
+    the row over.
+    """
+    renewal: Final = asyncio.create_task(_renew_claim_until_cancelled(context))
+    try:
+        if response is not None and response.status in _TERMINAL_STATUSES and response.usage is not None:
+            return await _bill(context, response)
+        return await _release(context, "released")
+    finally:
+        renewal.cancel()
+
+
 async def poll_and_log_background_interaction_cost(
     context: BackgroundInteractionPollContext,
     fetch_interaction: FetchInteraction = fetch_background_interaction,
 ) -> SettlementOutcome | None:
     last_response: InteractionsAPIResponse | None = None  # rebind-ok: the give-up path settles from the last poll
-    for interval in _poll_intervals(
+    await asyncio.sleep(context.start_delay_seconds)
+    intervals: Final = _poll_intervals(
         initial=context.initial_interval_seconds,
         maximum=context.max_interval_seconds,
         timeout=context.timeout_seconds,
-    ):
+    )
+    for interval in intervals:
         await asyncio.sleep(interval)
         if await _settled_elsewhere(context):
             return None
         try:
             response = await fetch_interaction(context)
         except Exception as e:  # noqa: BLE001  # any fetch error must not kill the billing poll loop
+            if context.resumed and _is_permanent_fetch_error(e):
+                verbose_logger.warning(
+                    "Stopped fetching resumed background interaction %s: the provider answered %s, so this "
+                    "process cannot read it and leaves it to the replica that created it until the poll window ends",
+                    context.interaction_id,
+                    e,
+                )
+                if await _settled_elsewhere_before_the_window_ends(context, intervals):
+                    return None
+                break
             verbose_logger.debug(
                 "Background interaction cost poll for %s failed, will retry: %s",
                 context.interaction_id,
@@ -363,11 +531,11 @@ async def poll_and_log_background_interaction_cost(
             continue
         if not claimed:
             return None
-        return await _record(context, await _settle_terminal(logging_obj=context.logging_obj, response=response))
+        return await _settle(context, response)
     if not await _claim(context):
         return None
     if last_response is not None and last_response.status in _TERMINAL_STATUSES:
-        return await _record(context, await _settle_terminal(logging_obj=context.logging_obj, response=last_response))
+        return await _settle(context, last_response)
     if last_response is not None and last_response.status not in _POLLABLE_STATUSES:
         verbose_logger.error(
             "Gave up cost polling for background interaction %s after %ss: its last status %r is in neither "
@@ -379,20 +547,10 @@ async def poll_and_log_background_interaction_cost(
         )
     else:
         verbose_logger.warning(
-            "Gave up cost polling for background interaction %s after %ss; its usage will not be tracked",
+            "Gave up cost polling for background interaction %s; its usage will not be tracked",
             context.interaction_id,
-            context.timeout_seconds,
         )
-    await _release_open_budget_reservation(logging_obj=context.logging_obj)
-    return await _record(context, "unsettled")
-
-
-async def _settle_terminal(logging_obj: "LiteLLMLoggingObj", response: InteractionsAPIResponse) -> SettlementOutcome:
-    if response.status in _TERMINAL_STATUSES and response.usage is not None:
-        await _bill_settled_interaction(logging_obj=logging_obj, response=response)
-        return "billed"
-    await _release_open_budget_reservation(logging_obj=logging_obj)
-    return "released"
+    return await _release(context, "unsettled")
 
 
 async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") -> None:
@@ -405,7 +563,9 @@ async def _release_open_budget_reservation(logging_obj: "LiteLLMLoggingObj") ->
     settlement must release the reservation here or the spend counters stay
     pinned at the estimated cost.
     """
-    metadata: Final = get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
+    metadata: Final = _OBJECT_MAPPING.validate_python(
+        get_litellm_metadata_from_kwargs(kwargs=logging_obj.model_call_details)
+    )
     budget_reservation: Final = metadata.get("user_api_key_budget_reservation")
     if not isinstance(budget_reservation, dict):
         return
@@ -518,13 +678,18 @@ class _UnverifiedRegistrationStore:
             return False
         return await self.local.claim(interaction_id)
 
+    async def renew_claim(self, interaction_id: str) -> None:
+        if await self.local.is_claimed(interaction_id):
+            return
+        await self.durable.renew_claim(interaction_id)
+
     async def record_outcome(self, interaction_id: str, outcome: SettlementOutcome) -> None:
         if await self.local.is_claimed(interaction_id):
             return
         await self.durable.record_outcome(interaction_id, outcome)
 
-    async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]:
-        return await self.durable.unclaimed()
+    async def unsettled(self) -> Sequence[PendingBackgroundInteraction]:
+        return await self.durable.unsettled()
 
 
 async def _registered_store(
@@ -559,8 +724,6 @@ async def maybe_schedule_background_interaction_cost_polling(
     logging_obj: Final = create_kwargs.get("litellm_logging_obj")
     if not isinstance(logging_obj, Logging):
         return None
-    api_key: Final = create_kwargs.get("api_key")
-    api_base: Final = create_kwargs.get("api_base")
     pending: Final = PendingBackgroundInteraction(
         interaction_id=response.id,
         custom_llm_provider=custom_llm_provider,
@@ -571,8 +734,7 @@ async def maybe_schedule_background_interaction_cost_polling(
         interaction_id=response.id,
         custom_llm_provider=custom_llm_provider,
         logging_obj=logging_obj,
-        api_key=api_key if isinstance(api_key, str) else None,
-        api_base=api_base if isinstance(api_base, str) else None,
+        credentials=_fetch_credentials(create_kwargs),
         store=await _registered_store(store or _STORE.store, pending),
     )
     return _track_poll(context, fetch_interaction)
@@ -603,10 +765,7 @@ async def _settle_before_delete(
 ) -> SettlementOutcome | None:
     if not await _claim(context):
         return None
-    if response is None:
-        await _release_open_budget_reservation(logging_obj=context.logging_obj)
-        return await _record(context, "released")
-    return await _record(context, await _settle_terminal(logging_obj=context.logging_obj, response=response))
+    return await _settle(context, response)
 
 
 async def maybe_settle_background_interaction_before_delete(
@@ -622,14 +781,11 @@ async def maybe_settle_background_interaction_before_delete(
     pending: Final = await _pending(settlement_store, interaction_id)
     if pending is None:
         return None
-    api_key: Final = delete_kwargs.get("api_key")
-    api_base: Final = delete_kwargs.get("api_base")
     context: Final = BackgroundInteractionPollContext(
         interaction_id=interaction_id,
         custom_llm_provider=pending.custom_llm_provider,
         logging_obj=_rebuild_logging_obj(pending.create_context),
-        api_key=api_key if isinstance(api_key, str) else None,
-        api_base=api_base if isinstance(api_base, str) else None,
+        credentials=_fetch_credentials(delete_kwargs),
         store=settlement_store,
     )
     try:
@@ -644,9 +800,9 @@ async def maybe_settle_background_interaction_before_delete(
     return await _settle_before_delete(context, response)
 
 
-async def _unclaimed(store: BackgroundSettlementStore) -> Sequence[PendingBackgroundInteraction]:
+async def _unsettled(store: BackgroundSettlementStore) -> Sequence[PendingBackgroundInteraction]:
     try:
-        return await store.unclaimed()
+        return await store.unsettled()
     except Exception:  # noqa: BLE001  # an unreadable store at startup leaves its rows for the next boot
         verbose_logger.exception("Could not list the unsettled background interactions")
         return ()
@@ -662,17 +818,36 @@ class PollSchedule:
 DEFAULT_POLL_SCHEDULE: Final = PollSchedule()
 
 
+def _deployment_id(create_context: BackgroundInteractionCreateContext) -> str | None:
+    model_info: Final = create_context.metadata.get("model_info")
+    deployment_id: Final = model_info.get("id") if isinstance(model_info, dict) else None
+    return deployment_id if isinstance(deployment_id, str) else None
+
+
+def _model_group(create_context: BackgroundInteractionCreateContext) -> str | None:
+    model_group: Final = create_context.metadata.get("model_group")
+    return model_group if isinstance(model_group, str) else None
+
+
+def _seconds_until(moment: datetime | None) -> float:
+    if moment is None:
+        return 0.0
+    return max((moment - datetime.now(timezone.utc)).total_seconds(), 0.0)
+
+
 def _resumed_context(
     row: PendingBackgroundInteraction, store: BackgroundSettlementStore, schedule: PollSchedule
 ) -> BackgroundInteractionPollContext:
-    age_seconds: Final = (datetime.now(timezone.utc) - row.created_at).total_seconds()
     return BackgroundInteractionPollContext(
         interaction_id=row.interaction_id,
         custom_llm_provider=row.custom_llm_provider,
         logging_obj=_rebuild_logging_obj(row.create_context),
+        deployment_id=_deployment_id(row.create_context),
+        model_group=_model_group(row.create_context),
         initial_interval_seconds=schedule.initial_interval_seconds,
         max_interval_seconds=schedule.max_interval_seconds,
-        timeout_seconds=max(schedule.timeout_seconds - age_seconds, schedule.initial_interval_seconds),
+        timeout_seconds=schedule.timeout_seconds,
+        start_delay_seconds=_seconds_until(row.claim_expires_at),
         store=store,
         resumed=True,
     )
@@ -684,13 +859,15 @@ async def resume_unsettled_background_interactions(
     schedule: PollSchedule = DEFAULT_POLL_SCHEDULE,
 ) -> tuple["asyncio.Task[SettlementOutcome | None]", ...]:
     """
-    Pick up every settlement no process has claimed, which is what a replica
-    that died mid-poll leaves behind. Each resumed poll keeps the remaining
-    share of the original timeout and gets at least one fetch, so a completed
-    interaction is still billed however late the resume comes.
+    Pick up every settlement no process has finished, which is what a replica
+    that died mid-poll or mid-settlement leaves behind. A row whose claim is
+    still within its lease waits until the lease runs out, then polls like any
+    other. Each resumed poll gets the full poll window however old its row is,
+    so a fetch that fails right after a late restart is retried rather than
+    ending the row as unsettled.
     """
     return tuple(
         _track_poll(_resumed_context(row, store, schedule), fetch_interaction)
-        for row in await _unclaimed(store)
+        for row in await _unsettled(store)
         if row.interaction_id not in _ACTIVE_POLLS
     )

litellm/litellm_core_utils/litellm_logging.py

@@ -2295,6 +2295,36 @@ class Logging(LiteLLMLoggingBaseClass):
         self.model_call_details["combined_usage_object"] = usage
         self.model_call_details["response_cost"] = response_cost
 
+    def record_continued_partial(self, usage: Usage, response_cost: float, response_id: str) -> None:
+        """Bill what a cut stream already consumed on this continuation's success log, under the id the caller saw."""
+        self.model_call_details["continued_partial_usage"] = usage
+        self.model_call_details["continued_partial_cost"] = response_cost
+        self.model_call_details["continued_response_id"] = response_id
+
+    def _add_continued_partial_spend(self, response: object) -> None:
+        partial_usage: Final = self.model_call_details.get("continued_partial_usage")
+        partial_cost: Final = self.model_call_details.get("continued_partial_cost")
+        own_usage: Final = getattr(response, "usage", None)
+        own_cost: Final = self.model_call_details.get("response_cost")
+        if not (isinstance(partial_usage, Usage) and isinstance(partial_cost, float) and isinstance(own_usage, Usage)):
+            return
+        from litellm.cost_calculator import BaseTokenUsageProcessor
+
+        self.model_call_details["combined_usage_object"] = BaseTokenUsageProcessor.combine_usage_objects(
+            usage_objects=[own_usage, partial_usage]
+        )
+        if not isinstance(own_cost, (int, float)):
+            return
+        self.model_call_details["response_cost"] = own_cost + partial_cost
+        if self.cost_breakdown is not None:
+            self.cost_breakdown = self.cost_breakdown | CostBreakdown(
+                total_cost=own_cost + partial_cost,
+                additional_costs={
+                    **self.cost_breakdown.get("additional_costs", {}),
+                    "mid_stream_partial_cost": partial_cost,
+                },
+            )
+
     def record_assembled_response_for_failure(self, assembled: ModelResponse) -> None:
         """Bill a fully streamed response on the failure log when a post-call hook rejects it."""
         usage: Final = getattr(assembled, "usage", None)
@@ -2590,6 +2620,15 @@ class Logging(LiteLLMLoggingBaseClass):
             return response
         return response.model_copy(update={"model": model})
 
+    def _with_continued_response_id(
+        self,
+        response: ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse,
+    ) -> ModelResponse | TextCompletionResponse | ResponsesAPIResponse | InteractionsAPIResponse:
+        response_id: Final = self.model_call_details.get("continued_response_id")
+        if not isinstance(response_id, str) or getattr(response, "id", None) == response_id:
+            return response
+        return response.model_copy(update={"id": response_id})
+
     def _success_handler_helper_fn(
         self,
         result: object = None,
@@ -2908,8 +2947,11 @@ class Logging(LiteLLMLoggingBaseClass):
                 self.model_call_details["response_cost"] = self.response_cost_calculator(
                     result=complete_streaming_response
                 )
+                self._add_continued_partial_spend(complete_streaming_response)
                 self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
-                logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response)
+                logged_streaming_response: Final = self._with_continued_response_id(
+                    self._with_client_facing_stream_model(complete_streaming_response)
+                )
                 self.model_call_details["complete_streaming_response"] = logged_streaming_response
                 ## STANDARDIZED LOGGING PAYLOAD
                 self.model_call_details["standard_logging_object"] = self._build_standard_logging_payload(
@@ -3445,12 +3487,15 @@ class Logging(LiteLLMLoggingBaseClass):
                     self.model,
                 )
                 self.model_call_details["response_cost"] = None
+            self._add_continued_partial_spend(complete_streaming_response)
 
             self._merge_hidden_params_from_response_into_metadata(complete_streaming_response)
 
             await self._prepare_baseline_cache_estimate(complete_streaming_response)
 
-            logged_streaming_response: Final = self._with_client_facing_stream_model(complete_streaming_response)
+            logged_streaming_response: Final = self._with_continued_response_id(
+                self._with_client_facing_stream_model(complete_streaming_response)
+            )
             self.model_call_details["async_complete_streaming_response"] = logged_streaming_response
 
             ## STANDARDIZED LOGGING PAYLOAD

litellm/litellm_core_utils/llm_response_utils/get_headers.py

@@ -1,8 +1,11 @@
 from collections.abc import Mapping
 from typing import Final
 
+import httpx
 from pydantic import TypeAdapter, ValidationError
 
+from litellm.constants import PROVIDER_REQUEST_ID_HEADERS
+
 _HEADERS: Final = TypeAdapter(Mapping[str, str])
 _REQUEST_ID_HEADERS: Final = ("request-id", "x-request-id", "llm_provider-request-id", "llm_provider-x-request-id")
 
@@ -16,6 +19,16 @@ def get_provider_request_id(headers: object) -> str | None:
     return next((normalized[name] for name in _REQUEST_ID_HEADERS if normalized.get(name)), None)
 
 
+def provider_request_id_headers(headers: Mapping[str, object] | None) -> httpx.Headers | None:
+    """The provider request id headers in `headers` whose values are ASCII strings, or None when there are none."""
+    request_id_headers: Final = tuple(
+        (name, value)
+        for name, value in (headers or {}).items()
+        if name.lower() in PROVIDER_REQUEST_ID_HEADERS and isinstance(value, str) and value.isascii()
+    )
+    return httpx.Headers(request_id_headers) if request_id_headers else None
+
+
 def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> dict:
     """
 

litellm/litellm_core_utils/model_param_helper.py

@@ -1,3 +1,4 @@
+from collections.abc import Mapping
 from functools import lru_cache
 from typing import ClassVar, Final
 
@@ -22,6 +23,24 @@ from litellm.types.llms.anthropic import AnthropicMessagesRequest
 from litellm.types.rerank import RerankRequest
 
 
+def peek_reasoning_summary_aliases(optional_params: Mapping[str, object]) -> object | None:
+    """Read AI-SDK-style reasoning summary from optional_params or nested extra_body.
+
+    Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped.
+    """
+    if "reasoningSummary" in optional_params:
+        return optional_params["reasoningSummary"]
+    if "reasoning_summary" in optional_params:
+        return optional_params["reasoning_summary"]
+    extra_body: Final = optional_params.get("extra_body")
+    if isinstance(extra_body, dict):
+        if "reasoningSummary" in extra_body:
+            return extra_body["reasoningSummary"]
+        if "reasoning_summary" in extra_body:
+            return extra_body["reasoning_summary"]
+    return None
+
+
 class ModelParamHelper:
     # Cached at class level — deterministic set built from static OpenAI type annotations
     relevant_logging_args: ClassVar[frozenset[str]] = frozenset()

litellm/litellm_core_utils/prompt_templates/common_utils.py

@@ -1894,6 +1894,11 @@ def strip_litellm_internal_message_fields(message: AllMessageValues) -> AllMessa
     return _message_without_fields(message, LITELLM_INTERNAL_MESSAGE_FIELDS)
 
 
+def strip_prefill_prefix_field(message: AllMessageValues) -> AllMessageValues:
+    """Drop the ``prefix`` flag Mistral and DeepSeek read off a prefill, for endpoints that reject unknown keys."""
+    return _message_without_fields(message, frozenset({"prefix"}))
+
+
 def filter_value_from_dict(dictionary: dict, key: str, depth: int = 0) -> Any:
     """
     Filters a value from a dictionary

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -9,10 +9,11 @@ import xml.etree.ElementTree as ET
 from collections.abc import Container, Iterable, Iterator, Mapping, Sequence
 from enum import Enum
 from types import MappingProxyType
-from typing import Any, Final, TypeAlias, TypedDict, cast, overload
+from typing import Annotated, Any, Final, TypeAlias, TypedDict, cast, overload
 
 from jinja2.sandbox import ImmutableSandboxedEnvironment
-from pydantic import BaseModel, JsonValue, TypeAdapter
+from pydantic import BaseModel, BeforeValidator, ConfigDict, JsonValue, TypeAdapter, ValidationError, model_validator
+from pydantic_core import ErrorDetails, PydanticCustomError
 
 import litellm
 import litellm.types
@@ -2582,6 +2583,48 @@ def _with_native_redacted_data(
 _AnthropicMessageList: TypeAlias = list[AllAnthropicPassThroughMessageValues]
 
 
+def _client_tool_result(message: AllMessageValues, replaced_server_tool_answers: frozenset[str]) -> bool:
+    return (
+        message_field(message, "role") in ("tool", "function")
+        and message_field(message, "tool_call_id") not in replaced_server_tool_answers
+    )
+
+
+def _tool_result_later_in_turn(
+    messages: Sequence[AllMessageValues], replaced_server_tool_answers: frozenset[str]
+) -> tuple[bool, ...]:
+    from_the_end: Final = itertools.accumulate(
+        reversed(messages),
+        lambda later, message: (
+            message_field(message, "role") in ("user", "tool", "function")
+            and (later or _client_tool_result(message, replaced_server_tool_answers))
+        ),
+        initial=False,
+    )
+    return tuple(from_the_end)[::-1][1:]
+
+
+def _empty_text_placeholder(message: AllMessageValues) -> AnthropicMessagesTextParam:
+    placeholder: Final = AnthropicMessagesTextParam(type="text", text=_EMPTY_TEXT_PLACEHOLDER)
+    add_cache_control_to_content(anthropic_content_element=placeholder, original_content_element=message)
+    return placeholder
+
+
+def _cache_breakpoint_placeholder(
+    message: AllMessageValues, tool_result_follows: bool
+) -> tuple[AnthropicMessagesTextParam, ...]:
+    if tool_result_follows or not message_field(message, "cache_control"):
+        return ()
+    return (_empty_text_placeholder(message),)
+
+
+def _empty_user_turn_placeholder(span: Sequence[AllMessageValues]) -> AnthropicMessagesTextParam | None:
+    empty_user_message: Final = next(
+        (message for message in reversed(span) if message["role"] == "user" and not message.get("content")), None
+    )
+    return None if empty_user_message is None else _empty_text_placeholder(empty_user_message)
+
+
 def anthropic_messages_pt(
     messages: list[AllMessageValues],
     model: str,
@@ -2636,6 +2679,7 @@ def anthropic_messages_pt(
     is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
     force_base64 = is_bedrock_invoke or is_vertex_ai
 
+    tool_result_later_in_turn: Final = _tool_result_later_in_turn(messages, replaced_server_tool_answers)
     msg_i = 0
     while msg_i < len(messages):
         user_content: list[AnthropicMessagesUserMessageValues] = []
@@ -2648,8 +2692,12 @@ def anthropic_messages_pt(
                 ChatCompletionToolMessage | ChatCompletionUserMessage | ChatCompletionFunctionMessage
             ) = messages[msg_i]
             if user_message_types_block["role"] == "user":
-                if isinstance(user_message_types_block["content"], list):
-                    for m in user_message_types_block["content"]:
+                if not (user_block_content := user_message_types_block.get("content")):
+                    user_content.extend(
+                        _cache_breakpoint_placeholder(user_message_types_block, tool_result_later_in_turn[msg_i])
+                    )
+                elif isinstance(user_block_content, list):
+                    for m in user_block_content:
                         if m.get("type", "") == "image_url":
                             m = cast(ChatCompletionImageObject, m)
                             format = m["image_url"].get("format") if isinstance(m["image_url"], dict) else None
@@ -2720,10 +2768,10 @@ def anthropic_messages_pt(
                                     _file_content_element,
                                 )
                             )
-                elif isinstance(user_message_types_block["content"], str):
+                elif isinstance(user_block_content, str):
                     _anthropic_content_text_element: AnthropicMessagesTextParam = {
                         "type": "text",
-                        "text": user_message_types_block["content"],
+                        "text": user_block_content,
                     }
                     _content_element = add_cache_control_to_content(
                         anthropic_content_element=_anthropic_content_text_element,
@@ -2747,6 +2795,8 @@ def anthropic_messages_pt(
 
         if user_content:
             new_messages.append({"role": "user", "content": user_content})
+        elif (placeholder := _empty_user_turn_placeholder(messages[init_msg_i:msg_i])) is not None:
+            new_messages.append({"role": "user", "content": [placeholder]})
 
         # Track unique tool IDs in this merge block to avoid duplication
         unique_tool_ids: set[str] = set()
@@ -4253,6 +4303,24 @@ def _with_text_when_document_only(message: BedrockMessageBlock) -> BedrockMessag
     return BedrockMessageBlock(role="user", content=[*blocks[:cut], placeholder, *blocks[cut:]])
 
 
+def _rstrip_text_block(block: BedrockContentBlock) -> BedrockContentBlock:
+    return BedrockContentBlock(text=block["text"].rstrip()) if "text" in block else block
+
+
+def _assistant_turn_rstripped(message: BedrockMessageBlock) -> BedrockMessageBlock:
+    """Claude on Converse rejects a final assistant turn that ends in whitespace, as the Anthropic path strips it."""
+    if message["role"] != "assistant":
+        return message
+    blocks: Final = message["content"]
+    last_text_index: Final = max((index for index, block in enumerate(blocks) if "text" in block), default=None)
+    return BedrockMessageBlock(
+        role="assistant",
+        content=[
+            _rstrip_text_block(block) if index == last_text_index else block for index, block in enumerate(blocks)
+        ],
+    )
+
+
 def _ensure_document_messages_have_text(
     contents: list[BedrockMessageBlock],
 ) -> list[BedrockMessageBlock]:
@@ -4844,7 +4912,12 @@ class BedrockConverseMessagesProcessor:
                     llm_provider=llm_provider,
                 )
 
-        return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
+        converse_messages: Final = _ensure_document_messages_have_text(
+            _rename_duplicate_bedrock_document_names(contents)
+        )
+        if not (to_claude and converse_messages):
+            return converse_messages
+        return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])]
 
     @staticmethod
     def translate_thinking_blocks_to_reasoning_content_blocks(
@@ -5253,7 +5326,10 @@ def _bedrock_converse_messages_pt(
                 llm_provider=llm_provider,
             )
 
-    return _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
+    converse_messages: Final = _ensure_document_messages_have_text(_rename_duplicate_bedrock_document_names(contents))
+    if not (to_claude and converse_messages):
+        return converse_messages
+    return [*converse_messages[:-1], _assistant_turn_rstripped(converse_messages[-1])]
 
 
 def make_valid_bedrock_tool_name(input_tool_name: str) -> str:
@@ -5516,7 +5592,7 @@ def function_call_prompt(
     ]
 
 
-def response_schema_prompt(model: str, response_schema: dict) -> str:
+def response_schema_prompt(model: str, response_schema: dict, llm_provider: str) -> str:
     """
     Decides if a user-defined custom prompt or default needs to be used
 
@@ -5532,11 +5608,8 @@ def response_schema_prompt(model: str, response_schema: dict) -> str:
         custom_prompt_details = litellm.custom_prompt_dict["response_schema_prompt"]
 
     if custom_prompt_details is not None:
-        return custom_prompt(
-            role_dict=custom_prompt_details["roles"],
-            initial_prompt_value=custom_prompt_details["initial_prompt_value"],
-            final_prompt_value=custom_prompt_details["final_prompt_value"],
-            messages=response_schema_as_message,
+        return render_custom_prompt_template(
+            template=custom_prompt_details, messages=response_schema_as_message, model=model, llm_provider=llm_provider
         )
     else:
         return default_response_schema_prompt(response_schema=response_schema)
@@ -5556,8 +5629,95 @@ def default_response_schema_prompt(response_schema: dict) -> str:
 
 
 # Custom prompt template
+def _mapping_or_none(value: object) -> object:
+    if value is None or issubclass(type(value), Mapping):
+        return value
+    verbose_logger.warning(
+        "Ignoring a custom prompt template roles value of type %s because it is not a mapping, "
+        "so those messages render without role markers",
+        type(value).__name__,
+    )
+    return None
+
+
+class CustomPromptRole(BaseModel):
+    model_config = ConfigDict(frozen=True)
+
+    pre_message: str | None = None
+    post_message: str | None = None
+
+
+class CustomPromptRoles(BaseModel):
+    model_config = ConfigDict(frozen=True, title="custom prompt template")
+
+    roles: Mapping[str, Annotated[CustomPromptRole | None, BeforeValidator(_mapping_or_none)]]
+
+
+class CustomPromptTemplate(BaseModel):
+    model_config = ConfigDict(frozen=True, title="custom prompt template")
+
+    roles: Annotated[Mapping[object, object] | None, BeforeValidator(_mapping_or_none)] = None
+    initial_prompt_value: str | None = None
+    final_prompt_value: str | None = None
+
+    @model_validator(mode="after")
+    def _sets_a_rendered_field(self) -> "CustomPromptTemplate":
+        if self.roles is None and self.initial_prompt_value is None and self.final_prompt_value is None:
+            raise PydanticCustomError(
+                "empty_custom_prompt_template",
+                "it sets none of roles, initial_prompt_value, or final_prompt_value",
+            )
+        return self
+
+
+def render_custom_prompt_template(
+    template: Mapping[str, object],
+    messages: Sequence[Mapping[str, object]],
+    model: str,
+    llm_provider: str,
+) -> str:
+    try:
+        return _render_custom_prompt_template(template=template, messages=messages)
+    except ValidationError as error:
+        raise litellm.APIConnectionError(
+            message=f"The custom prompt template for {model} is invalid: {_describe_template_errors(error)}",
+            model=model,
+            llm_provider=llm_provider,
+        ) from error
+
+
+def _describe_template_errors(error: ValidationError) -> str:
+    return "; ".join(
+        _describe_template_error(detail) for detail in error.errors(include_url=False, include_input=False)
+    )
+
+
+def _describe_template_error(detail: ErrorDetails) -> str:
+    location: Final = ".".join(str(part) for part in detail["loc"])
+    return f"{location}: {detail['msg']}" if location else detail["msg"]
+
+
+def _render_custom_prompt_template(template: Mapping[str, object], messages: Sequence[Mapping[str, object]]) -> str:
+    validated: Final = CustomPromptTemplate.model_validate(template)
+    message_roles: Final = frozenset(message.get("role") for message in messages)
+    template_roles: Final = validated.roles or {}
+    used_roles: Final = CustomPromptRoles.model_validate(
+        {"roles": {role: entry for role, entry in template_roles.items() if role in message_roles}}
+    )
+    return custom_prompt(
+        role_dict={
+            role: {"pre_message": entry.pre_message or "", "post_message": entry.post_message or ""}
+            for role, entry in used_roles.roles.items()
+            if entry is not None
+        },
+        initial_prompt_value=validated.initial_prompt_value or "",
+        final_prompt_value=validated.final_prompt_value or "",
+        messages=list(messages),
+    )
+
+
 def custom_prompt(
-    role_dict: dict,
+    role_dict: Mapping[str, Mapping[str, str]],
     messages: list,
     initial_prompt_value: str = "",
     final_prompt_value: str = "",

litellm/litellm_core_utils/prompt_templates/mid_conversation_system.py

@@ -414,10 +414,11 @@ def _assistant_renders(message: object, answered: frozenset[str]) -> bool:
 def _renders(message: object, answered: frozenset[str]) -> bool:
     """Whether ``anthropic_messages_pt`` puts a block on the wire for this message.
 
-    A tool message always becomes a ``tool_result`` and a user message with string
-    content always becomes a text block (empty text gets a placeholder). A user list
-    renders only through parts of a type the converter emits; ``None``, an empty list,
-    and a list of other parts vanish. Assistant messages follow ``_assistant_renders``.
+    A tool message always becomes a ``tool_result``, and a user message with string or
+    missing content (``None``, an empty list) always puts its turn on the wire, an empty
+    turn getting a placeholder. A user list renders only through parts of a type the
+    converter emits, so a list of only other parts vanishes. Assistant messages follow
+    ``_assistant_renders``.
     """
     role: Final = message_field(message, "role")
     if role in _TOOL_ROLES:
@@ -425,8 +426,10 @@ def _renders(message: object, answered: frozenset[str]) -> bool:
     if role == "assistant":
         return _assistant_renders(message, answered)
     content: Final = message_field(message, "content")
-    return isinstance(content, str) or any(
-        message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content)
+    return (
+        isinstance(content, str)
+        or not content
+        or any(message_field(part, "type") in _RENDERED_PART_TYPES for part in parts_of(content))
     )
 
 

litellm/litellm_core_utils/realtime_streaming.py

@@ -1,16 +1,18 @@
 import asyncio
 import json
 import traceback
-from collections.abc import Coroutine, Mapping, Sequence
+from collections.abc import Callable, Coroutine, Mapping, Sequence
 from dataclasses import dataclass
 from enum import Enum, auto
-from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypedDict, cast
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Any, Final, NoReturn, Protocol, TypeAlias, TypedDict, cast
 
 from pydantic import TypeAdapter
 from typing_extensions import ReadOnly
 
 import litellm
 from litellm._logging import redact_internal_details_from_client_message, verbose_logger
+from litellm._uuid import uuid
 from litellm.constants import (
     REALTIME_METERED_USAGE_KEY,
     REALTIME_SESSION_FAILURE_LOGGED_KEY,
@@ -116,10 +118,87 @@ def _decode_json_object(payload: str) -> Mapping[str, object]:
     return json.loads(payload)
 
 
+_GUARDRAIL_TURN_DETECTION_GATE: Final[Mapping[str, object]] = MappingProxyType(
+    {"type": "server_vad", "create_response": False}
+)
+_EMPTY_SESSION: Final[Mapping[str, object]] = MappingProxyType({})
+_JSON_OBJECT: Final = TypeAdapter(Mapping[str, object])
+_GUARDRAIL_EVENT_ID_PREFIX: Final = "litellm_guardrail_"
+
+
+def _json_object(value: object) -> Mapping[str, object] | None:
+    return _JSON_OBJECT.validate_python(value) if isinstance(value, Mapping) else None
+
+
+def _mapping_field(container: Mapping[str, object], key: str) -> Mapping[str, object] | None:
+    return _json_object(container.get(key))
+
+
+def _guardrail_event_id() -> str:
+    return f"{_GUARDRAIL_EVENT_ID_PREFIX}{uuid.uuid4().hex}"
+
+
+def _error_event_id(event: Mapping[str, object]) -> str | None:
+    error: Final = _mapping_field(event, "error")
+    caused_by: Final = None if error is None else error.get("event_id")
+    return caused_by if isinstance(caused_by, str) else None
+
+
+_TurnDetectionGate: TypeAlias = Callable[[object], Mapping[str, object]]
+
+
+def _turn_detection_with_auto_response_off(turn_detection: object) -> Mapping[str, object]:
+    client_turn_detection: Final = _json_object(turn_detection)
+    if client_turn_detection is None:
+        return dict(_GUARDRAIL_TURN_DETECTION_GATE)
+    return {**client_turn_detection, "create_response": False}
+
+
+def _nested_turn_detection_part(session: Mapping[str, object], gate: _TurnDetectionGate) -> Mapping[str, object]:
+    audio: Final = _mapping_field(session, "audio")
+    if audio is None:
+        return {}
+    audio_input: Final = _mapping_field(audio, "input")
+    if audio_input is None or "turn_detection" not in audio_input:
+        return {}
+    gated_input: Final = {
+        **audio_input,
+        "turn_detection": gate(audio_input["turn_detection"]),
+    }
+    return {"audio": {**audio, "input": gated_input}}
+
+
+def _with_create_response_disabled(
+    session: Mapping[str, object], gate: _TurnDetectionGate
+) -> Mapping[str, object] | None:
+    """``session`` with every turn_detection the client sent unable to auto-respond, or None when it sent none."""
+    flat_part: Final = {"turn_detection": gate(session["turn_detection"])} if "turn_detection" in session else {}
+    nested_part: Final = _nested_turn_detection_part(session, gate)
+    if not flat_part and not nested_part:
+        return None
+    return {**session, **flat_part, **nested_part}
+
+
+def _gate_session_update(
+    session: object, gate: _TurnDetectionGate, inject_gate_when_absent: bool
+) -> Mapping[str, object] | None:
+    """The session to forward under a transcript guardrail, or None when the frame needs no change."""
+    client_session: Final = _json_object(session)
+    if client_session is None:
+        return None
+    gated: Final = _with_create_response_disabled(client_session, gate)
+    if gated is not None or not inject_gate_when_absent:
+        return gated
+    return {**client_session, "turn_detection": gate(None)}
+
+
 class RealtimeEventNormalizer(Protocol):
     def should_drop(self, event: object) -> bool: ...
     def normalize(self, event: dict) -> dict: ...
     def patch_outgoing_session(self, session: dict) -> dict: ...
+    def gate_turn_detection(self, turn_detection: object, /) -> Mapping[str, object]: ...
+    @property
+    def deletes_conversation_items(self) -> bool: ...
 
 
 class RealtimeUsageMeter(Protocol):
@@ -197,6 +276,8 @@ class RealTimeStreaming:
         # Track whether we have already sent the guardrail turn-detection update
         # that disables provider auto-response for transcription guardrails.
         self._guardrail_turn_detection_update_sent: bool = False
+        self._session_gate_sent: bool = False
+        self._guardrail_cancel_event_ids: frozenset[str] = frozenset()
         # Deferred Gemini Live setup: Pipecat may stream audio before session.update.
         # Buffer client audio until the backend acknowledges setup (setupComplete).
         self._backend_setup_complete: bool = provider_config is None or provider_config.requires_session_configuration()
@@ -213,6 +294,12 @@ class RealTimeStreaming:
         self._is_transcription_session: bool = force_transcription_model is not None
         # Optional per-provider GA event normalizer (e.g. XAIRealtimeNormalizer).
         self._event_normalizer = event_normalizer
+        self._gate_turn_detection: Final[_TurnDetectionGate] = (
+            _turn_detection_with_auto_response_off if event_normalizer is None else event_normalizer.gate_turn_detection
+        )
+        self._deletes_conversation_items: Final = (
+            event_normalizer is None or event_normalizer.deletes_conversation_items
+        )
         self._usage_meter = usage_meter
 
     @property
@@ -734,20 +821,37 @@ class RealTimeStreaming:
         except (json.JSONDecodeError, TypeError):
             return
 
+    def _make_turn_detection_update(self, turn_detection: Mapping[str, object]) -> dict[str, object]:
+        session: Final[dict[str, object]] = (
+            {"turn_detection": turn_detection}
+            if self._backend_uses_beta_protocol
+            else {"type": "realtime", "audio": {"input": {"turn_detection": turn_detection}}}
+        )
+        return {"type": "session.update", "session": session}
+
     def _make_disable_auto_response_message(self) -> str:
         """Return a session.update that disables VAD auto-response."""
-        turn_detection: Final[dict[str, str | bool]] = {
-            "type": "server_vad",
-            "create_response": False,
-        }
-        if self._backend_uses_beta_protocol:
-            session: dict[str, object] = {"turn_detection": turn_detection}
-        else:
-            session = {
-                "type": "realtime",
-                "audio": {"input": {"turn_detection": turn_detection}},
-            }
-        return json.dumps({"type": "session.update", "session": session})
+        return json.dumps(self._make_turn_detection_update(self._gate_turn_detection(None)))
+
+    async def _send_guardrail_cancel(self) -> None:
+        event_id: Final = _guardrail_event_id()
+        self._guardrail_cancel_event_ids = self._guardrail_cancel_event_ids | {event_id}
+        await self._send_to_backend(json.dumps({"type": "response.cancel", "event_id": event_id}))
+
+    async def _withdraw_blocked_turn(self, item_id: str | None) -> None:
+        await self._send_guardrail_cancel()
+        if item_id is not None and self._deletes_conversation_items:
+            await self._send_to_backend(json.dumps({"type": "conversation.item.delete", "item_id": item_id}))
+
+    def _is_guardrail_cancel_error(self, event: Mapping[str, object]) -> bool:
+        return _error_event_id(event) in self._guardrail_cancel_event_ids
+
+    async def _send_session_gate_once(self) -> None:
+        """Gate the backend's default turn_detection ahead of the first client session.update, which overrides it."""
+        if self._session_gate_sent or not self._should_disable_vad_auto_response():
+            return
+        self._session_gate_sent = True
+        await self._send_to_backend(self._make_disable_auto_response_message())
 
     async def _maybe_send_guardrail_turn_detection_update(self) -> None:
         """Disable provider auto-response once when transcription guardrails are enabled."""
@@ -921,7 +1025,7 @@ class RealTimeStreaming:
                 if pre_block_backend_message is not None:
                     await self._send_to_backend(pre_block_backend_message)
                 if not self._is_transcription_session:
-                    await self._send_to_backend(json.dumps({"type": "response.cancel"}))
+                    await self._withdraw_blocked_turn(item_id)
                 # Send the policy violation hint (shows as small gray status text in UI).
                 await self.websocket.send_text(
                     json.dumps(
@@ -1051,10 +1155,7 @@ class RealTimeStreaming:
                 self.store_message(event_str)
                 self._capture_transcription_usage(event)
                 await self._send_event_to_client(event, event_str)
-                blocked = await self.run_realtime_guardrails(
-                    cast(str, transcript),
-                    item_id=cast(str | None, event.get("item_id")),
-                )
+                blocked = await self.run_realtime_guardrails(cast(str, transcript))
                 if not blocked and self._should_disable_vad_auto_response():
                     await self._send_to_backend(json.dumps({"type": "response.create"}))
                 continue
@@ -1086,7 +1187,7 @@ class RealTimeStreaming:
         if event_type == "session.created" and self._should_disable_vad_auto_response():
             self.store_message(event_obj)
             await self.websocket.send_text(self._event_to_client_json(event_obj))
-            await self._send_to_backend(self._make_disable_auto_response_message())
+            await self._send_session_gate_once()
             return True
 
         if event_type == "conversation.item.input_audio_transcription.completed":
@@ -1140,6 +1241,10 @@ class RealTimeStreaming:
                 if self._usage_meter is not None:
                     self._usage_meter.observe_backend_event(event)
 
+                if event.get("type") == "error" and self._is_guardrail_cancel_error(event):
+                    verbose_logger.debug("Realtime: not forwarding an error caused by a guardrail event: %s", event)
+                    continue
+
                 if await self._handle_raw_backend_message(event, raw_response):
                     continue
 
@@ -1472,19 +1577,21 @@ class RealTimeStreaming:
                         and self.session_configuration_request is None
                         and not self._guardrail_turn_detection_update_sent
                         and self._should_disable_vad_auto_response()
-                    ):
-                        session: Mapping[str, object] | None = msg_obj.setdefault("session", {})
-                        if isinstance(session, dict):
-                            existing_td = session.get("turn_detection")
-                            if not isinstance(existing_td, dict):
-                                existing_td = {}
-                            existing_td["create_response"] = False
-                            session["turn_detection"] = existing_td
-                            message = json.dumps(msg_obj)
-                            guardrail_turn_detection_injected = True
-                            verbose_logger.debug(
-                                "Injected turn_detection into first session.update for audio transcription guardrails"
+                        and (
+                            gated_first := _gate_session_update(
+                                client_event.get("session", _EMPTY_SESSION),
+                                self._gate_turn_detection,
+                                inject_gate_when_absent=True,
                             )
+                        )
+                        is not None
+                    ):
+                        msg_obj["session"] = gated_first
+                        message = json.dumps(msg_obj)
+                        guardrail_turn_detection_injected = True
+                        verbose_logger.debug(
+                            "Injected turn_detection into first session.update for audio transcription guardrails"
+                        )
 
                     ## GUARDRAIL: Force ``create_response`` to False in any
                     # client-provided ``turn_detection`` so a later
@@ -1492,52 +1599,25 @@ class RealTimeStreaming:
                     # and bypass the transcription guardrail after the
                     # initial disable. Covers both the flat beta key and the
                     # nested GA ``audio.input.turn_detection`` shape, since
-                    # the GA remap below also accepts either form. Skipped
-                    # when the injection block above already ran for this
-                    # message, to avoid redundant double-serialization.
+                    # the GA remap below also accepts either form. A later
+                    # update with no turn_detection is forwarded as is: OpenAI
+                    # keeps the earlier disable for an unset field, and xAI
+                    # falls back to its manual default, which never auto-responds.
                     if (
                         msg_type == "session.update"
                         and not guardrail_turn_detection_injected
                         and self._should_disable_vad_auto_response()
+                        and (
+                            gated_later := _gate_session_update(
+                                client_event.get("session"),
+                                self._gate_turn_detection,
+                                inject_gate_when_absent=False,
+                            )
+                        )
+                        is not None
                     ):
-                        session = client_event.get("session")
-                        if isinstance(session, dict):
-                            td_overridden = False
-                            flat_td = session.get("turn_detection")
-                            flat_td_present = flat_td is not None
-                            if flat_td_present:
-                                if not isinstance(flat_td, dict):
-                                    flat_td = {}
-                                if flat_td.get("create_response") is not False:
-                                    flat_td["create_response"] = False
-                                    session["turn_detection"] = flat_td
-                                    td_overridden = True
-                            nested_td_present = False
-                            audio = session.get("audio")
-                            if isinstance(audio, dict):
-                                audio_input = audio.get("input")
-                                if isinstance(audio_input, dict):
-                                    nested_td = audio_input.get("turn_detection")
-                                    if nested_td is not None:
-                                        nested_td_present = True
-                                        if not isinstance(nested_td, dict):
-                                            nested_td = {}
-                                        if nested_td.get("create_response") is not False:
-                                            nested_td["create_response"] = False
-                                            audio_input["turn_detection"] = nested_td
-                                            td_overridden = True
-                            # Symmetric with the first-update injection block:
-                            # if the client omitted turn_detection entirely on
-                            # a subsequent session.update, still inject the
-                            # ``create_response: False`` override so the
-                            # transcription guardrail cannot be re-enabled by
-                            # any downstream merge that drops the original
-                            # disable.
-                            if not flat_td_present and not nested_td_present:
-                                session["turn_detection"] = {"create_response": False}
-                                td_overridden = True
-                            if td_overridden:
-                                message = json.dumps(msg_obj)
+                        msg_obj["session"] = gated_later
+                        message = json.dumps(msg_obj)
 
                     # GA compatibility: remap beta-style session fields only when
                     # the upstream is in GA mode. Beta upstreams expect the flat
@@ -1590,6 +1670,8 @@ class RealTimeStreaming:
                 # would permanently disable the injection if ``_send_to_backend``
                 # raised — neither this loop nor
                 # ``_maybe_send_guardrail_turn_detection_update`` would retry.
+                if msg_type == "session.update" and self.provider_config is None:
+                    await self._send_session_gate_once()
                 sent = await self._send_to_backend(message)
                 if guardrail_turn_detection_injected and sent:
                     self._guardrail_turn_detection_update_sent = True

litellm/litellm_core_utils/streaming_handler.py

@@ -242,6 +242,7 @@ class CustomStreamWrapper:
         self.completion_stream = completion_stream
         self.sent_first_chunk = False
         self.sent_last_chunk = False
+        self._failure_logging: asyncio.Task[None] | None = None
         self._closed = False
         self._stream_created_time: float = time.time()
 
@@ -269,6 +270,7 @@ class CustomStreamWrapper:
         self.holding_chunk = ""
         self.complete_response = ""
         self.response_uptil_now = ""
+        self._emitted_disqualifying_content = False
         _model_info: Final[dict] = litellm_params.model_info or {}
 
         _api_base: Final = get_api_base(
@@ -1129,6 +1131,7 @@ class CustomStreamWrapper:
         if self.merge_reasoning_content_in_choices is True:
             reasoning_content: Final = getattr(model_response.choices[0].delta, "reasoning_content", None)
             if reasoning_content:
+                self._emitted_disqualifying_content = True
                 if self.sent_first_thinking_block is False:
                     # Ensure content is not None before concatenation
                     if model_response.choices[0].delta.content is None:
@@ -1865,9 +1868,7 @@ class CustomStreamWrapper:
                     if response.choices:
                         choice = response.choices[0]
                         if isinstance(choice, StreamingChoices):
-                            self.response_uptil_now += choice.delta.get("content", "") or ""
-                        else:
-                            self.response_uptil_now += ""
+                            self._accumulate_streamed_delta(choice.delta)
                     self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
                     # HANDLE STREAM OPTIONS
                     self.chunks.append(response)
@@ -2084,9 +2085,7 @@ class CustomStreamWrapper:
                     if processed_chunk.choices:
                         choice = processed_chunk.choices[0]
                         if isinstance(choice, StreamingChoices):
-                            self.response_uptil_now += choice.delta.get("content", "") or ""
-                        else:
-                            self.response_uptil_now += ""
+                            self._accumulate_streamed_delta(choice.delta)
                     self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
                     # Add mcp_list_tools to first chunk if present
                     if not self.sent_first_chunk and processed_chunk.choices:
@@ -2156,9 +2155,7 @@ class CustomStreamWrapper:
 
                         choice = processed_chunk.choices[0]
                         if isinstance(choice, StreamingChoices):
-                            self.response_uptil_now += choice.delta.get("content", "") or ""
-                        else:
-                            self.response_uptil_now += ""
+                            self._accumulate_streamed_delta(choice.delta)
                         self.rules.post_call_rules(input=self.response_uptil_now, model=self.model)
                         # RETURN RESULT
                         self.chunks.append(processed_chunk)
@@ -2172,7 +2169,7 @@ class CustomStreamWrapper:
             if self.logging_obj is not None:
                 self._record_partial_usage_for_failure()
                 ## LOGGING
-                asyncio.create_task(
+                self._failure_logging = asyncio.create_task(
                     self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
                 )
             await self.aclose()
@@ -2305,11 +2302,17 @@ class CustomStreamWrapper:
         if self.logging_obj is not None:
             self._record_partial_usage_for_failure()
             ## LOGGING
-            asyncio.create_task(
+            self._failure_logging = asyncio.create_task(
                 self.logging_obj.dispatch_failure_handlers(e, traceback_exception, prefer_async_handlers=True)
             )
         self._handle_stream_fallback_error(e)
 
+    async def wait_for_failure_logging(self, timeout: float) -> None:
+        """Let this stream's failure log read its own deployment before a fallback reuses the logging object."""
+        if self._failure_logging is None:
+            return
+        await asyncio.wait((self._failure_logging,), timeout=timeout)
+
     def _record_partial_usage_for_failure(self) -> None:
         """
         A stream that breaks mid-flight still billed the provider for the chunks
@@ -2346,6 +2349,35 @@ class CustomStreamWrapper:
                 recover_error,
             )
 
+    # Delta fields a text-only prefill continuation cannot carry, so a stream
+    # that emitted any of them is not eligible for mid-stream continuation.
+    _CONTINUATION_DISQUALIFYING_DELTA_FIELDS: Final = (
+        "tool_calls",
+        "function_call",
+        "reasoning_content",
+        "thinking_blocks",
+        "reasoning_items",
+        "audio",
+        "images",
+        "annotations",
+    )
+
+    @classmethod
+    def _delta_disqualifies_continuation(cls, delta: object) -> bool:
+        get: Final = getattr(delta, "get", None)
+        if not callable(get):
+            return False
+        return any(get(field) for field in cls._CONTINUATION_DISQUALIFYING_DELTA_FIELDS)
+
+    def _accumulate_streamed_delta(self, delta: object) -> None:
+        # Shared by the sync, async, and non-aiohttp iteration sites so answer
+        # text and the disqualifying-content latch stay in step across all three.
+        get: Final = getattr(delta, "get", None)
+        content: Final = get("content") if callable(get) else None
+        self.response_uptil_now += content if isinstance(content, str) else ""
+        if not self._emitted_disqualifying_content and self._delta_disqualifies_continuation(delta):
+            self._emitted_disqualifying_content = True
+
     def _handle_stream_fallback_error(self, e: Exception) -> "NoReturn":
         """
         Common error handling for both __next__ and __anext__.
@@ -2377,6 +2409,7 @@ class CustomStreamWrapper:
                 original_exception=e.original_exception,
                 generated_content=self.response_uptil_now,
                 is_pre_first_chunk=not self.sent_first_chunk,
+                emitted_disqualifying_content=self._emitted_disqualifying_content or e.emitted_disqualifying_content,
             )
 
         # Map to OpenAI exception format. Some providers' mappers (e.g.
@@ -2437,6 +2470,7 @@ class CustomStreamWrapper:
             original_exception=mapped_exception,
             generated_content=self.response_uptil_now,
             is_pre_first_chunk=not self.sent_first_chunk,
+            emitted_disqualifying_content=self._emitted_disqualifying_content,
         )
 
     @staticmethod

litellm/llms/anthropic/completion/transformation.py

@@ -14,8 +14,8 @@ import httpx
 import litellm
 from litellm.constants import DEFAULT_MAX_TOKENS
 from litellm.litellm_core_utils.prompt_templates.factory import (
-    custom_prompt,
     prompt_factory,
+    render_custom_prompt_template,
 )
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.chat.transformation import (
@@ -244,11 +244,8 @@ class AnthropicTextConfig(BaseConfig):
         if model in custom_prompt_dict:
             # check if the model has a registered custom prompt
             model_prompt_details: Final = custom_prompt_dict[model]
-            prompt = custom_prompt(
-                role_dict=model_prompt_details["roles"],
-                initial_prompt_value=model_prompt_details["initial_prompt_value"],
-                final_prompt_value=model_prompt_details["final_prompt_value"],
-                messages=messages,
+            prompt = render_custom_prompt_template(
+                template=model_prompt_details, messages=messages, model=model, llm_provider="anthropic"
             )
         else:
             prompt = prompt_factory(model=model, messages=messages, custom_llm_provider="anthropic")

litellm/llms/anthropic/count_tokens/token_counter.py

@@ -6,6 +6,7 @@ from typing import Any, Final
 
 from pydantic import JsonValue
 
+import litellm
 from litellm._logging import verbose_logger
 from litellm.exceptions import AuthenticationError
 from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
@@ -61,14 +62,24 @@ class AnthropicTokenCounter(BaseTokenCounter):
 
         try:
             auth_header: Final = await AnthropicModelInfo.aget_auth_header(
-                api_key=litellm_params.get("api_key"),
+                api_key=litellm_params.get("api_key") or litellm.anthropic_key or litellm.api_key,
                 api_base=api_base,
                 litellm_params=litellm_params,
                 allow_workload_identity=True,
             )
             if auth_header is None:
-                verbose_logger.warning("No Anthropic credential found for token counting")
-                return None
+                return TokenCountResponse(
+                    total_tokens=0,
+                    request_model=request_model,
+                    model_used=model_to_use,
+                    tokenizer_type="anthropic_api",
+                    error=True,
+                    error_message=(
+                        f"No Anthropic API key is set for model {request_model or model_to_use}, so tokens can't be "
+                        "counted. Set api_key on the deployment or ANTHROPIC_API_KEY in the proxy environment."
+                    ),
+                    status_code=401,
+                )
 
             result: Final = await anthropic_count_tokens_handler.handle_count_tokens_request(
                 model=model_to_use,

litellm/llms/anthropic/pass_through/adapters/handler.py

@@ -1,4 +1,5 @@
 from collections.abc import AsyncIterator, Coroutine, Iterator, Mapping, Sequence
+from dataclasses import dataclass
 from typing import (
     TYPE_CHECKING,
     Final,
@@ -62,6 +63,12 @@ class _CompletionKwargs(TypedDict, total=False, extra_items=object):
     custom_llm_provider: str
 
 
+@dataclass(frozen=True, slots=True)
+class _ChatTargetReasoning:
+    effort: str
+    summary: str | None = None
+
+
 def _messages_have_compaction_block(messages: _AnthropicMessages) -> bool:
     """Return True when any message carries a ``compaction`` content block."""
     for msg in messages:
@@ -441,15 +448,33 @@ class LiteLLMMessagesToCompletionTransformationHandler:
                     completion_kwargs["reasoning_effort"] = updated_reasoning_effort
 
     @staticmethod
-    def _plain_effort_for_chat_target(
+    def _requested_reasoning_summary(thinking: Mapping[str, object]) -> str | None:
+        if thinking.get("type") == "disabled":
+            return None
+        summary: Final = thinking.get("summary")
+        if isinstance(summary, str):
+            return summary
+        display: Final = thinking.get("display")
+        return "auto" if display is not None and display != "omitted" else None
+
+    @staticmethod
+    def _chat_target_reasoning(
         completion_kwargs: _CompletionKwargs,
         *,
         thinking: Mapping[str, object] | None,
-    ) -> str | None:
-        reasoning_effort: Final = completion_kwargs.get("reasoning_effort")
-        if not thinking or not isinstance(reasoning_effort, dict) or "summary" not in reasoning_effort:
+    ) -> _ChatTargetReasoning | None:
+        if not thinking:
             return None
-        effort: Final = reasoning_effort.get("effort")
+        reasoning_effort: Final = completion_kwargs.get("reasoning_effort")
+        wraps_summary: Final = isinstance(reasoning_effort, dict) and "summary" in reasoning_effort
+        requested_summary: Final = LiteLLMMessagesToCompletionTransformationHandler._requested_reasoning_summary(
+            thinking
+        )
+        if not wraps_summary and requested_summary is None:
+            return None
+        effort: Final[object] = (
+            reasoning_effort.get("effort") if isinstance(reasoning_effort, dict) else reasoning_effort
+        )
         model: Final = completion_kwargs.get("model")
         if not isinstance(effort, str) or not isinstance(model, str) or not model:
             return None
@@ -467,7 +492,12 @@ class LiteLLMMessagesToCompletionTransformationHandler:
             return None
         if resolved_provider == "litellm_proxy":
             return None
-        from litellm.responses.bridge_check import responses_api_bridge_check
+        from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary, responses_api_bridge_check
+
+        if requested_summary is not None and chat_target_accepts_reasoning_summary(local_model, resolved_provider):
+            return _ChatTargetReasoning(effort=effort, summary=requested_summary)
+        if not wraps_summary or not isinstance(reasoning_effort, dict):
+            return None
 
         web_search_options: Final = completion_kwargs.get("web_search_options")
         tools: Final = completion_kwargs.get("tools")
@@ -481,7 +511,7 @@ class LiteLLMMessagesToCompletionTransformationHandler:
             reasoning_effort=reasoning_effort,
             api_base=resolved_api_base,
         )
-        return None if model_info.get("mode") == "responses" else effort
+        return None if model_info.get("mode") == "responses" else _ChatTargetReasoning(effort=effort)
 
     @staticmethod
     def _normalize_reasoning_effort(
@@ -647,12 +677,14 @@ class LiteLLMMessagesToCompletionTransformationHandler:
             thinking=thinking,
         )
 
-        plain_effort: Final = LiteLLMMessagesToCompletionTransformationHandler._plain_effort_for_chat_target(
+        chat_reasoning: Final = LiteLLMMessagesToCompletionTransformationHandler._chat_target_reasoning(
             completion_kwargs,
             thinking=thinking,
         )
-        if plain_effort is not None:
-            completion_kwargs["reasoning_effort"] = plain_effort
+        if chat_reasoning is not None:
+            completion_kwargs["reasoning_effort"] = chat_reasoning.effort
+        if chat_reasoning is not None and chat_reasoning.summary is not None:
+            completion_kwargs["reasoning_summary"] = chat_reasoning.summary
 
         return completion_kwargs, tool_name_mapping
 

litellm/llms/azure_ai/anthropic/count_tokens/handler.py

@@ -30,7 +30,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
         self,
         model: str,
         messages: list[dict[str, JsonValue]],
-        api_key: str,
+        api_key: str | None,
         api_base: str,
         litellm_params: dict[str, object] | None = None,
         timeout: float | httpx.Timeout | None = None,
@@ -80,6 +80,15 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
                 api_key=api_key,
                 litellm_params=litellm_params,
             )
+            if not {"x-api-key", "Authorization"} & headers.keys():
+                raise AnthropicError(
+                    status_code=401,
+                    message=(
+                        f"No Azure AI API key or Entra ID credential is set for model {model}, so tokens can't be "
+                        "counted. Set api_key or an Entra ID credential on the deployment, or AZURE_AI_API_KEY or "
+                        "AZURE_API_KEY in the proxy environment."
+                    ),
+                )
 
             # Use LiteLLM's async httpx client
             async_client: Final = get_async_httpx_client(llm_provider=litellm.LlmProviders.AZURE_AI)

litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py

@@ -7,6 +7,7 @@ from typing import Any, Final
 
 from pydantic import JsonValue
 
+import litellm
 from litellm._logging import verbose_logger
 from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
     AzureAIAnthropicCountTokensHandler,
@@ -18,6 +19,21 @@ from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCou
 azure_ai_anthropic_count_tokens_handler: Final = AzureAIAnthropicCountTokensHandler()
 
 
+def _missing_api_base_response(request_model: str, model_to_use: str) -> TokenCountResponse:
+    return TokenCountResponse(
+        total_tokens=0,
+        request_model=request_model,
+        model_used=model_to_use,
+        tokenizer_type="azure_ai_anthropic_api",
+        error=True,
+        error_message=(
+            f"No Azure AI API base is set for model {request_model or model_to_use}, so tokens can't be counted. "
+            "Set api_base on the deployment or AZURE_AI_API_BASE or AZURE_API_BASE in the proxy environment."
+        ),
+        status_code=500,
+    )
+
+
 class AzureAIAnthropicTokenCounter(BaseTokenCounter):
     """Token counter implementation for Azure AI Anthropic provider using the CountTokens API."""
 
@@ -65,18 +81,15 @@ class AzureAIAnthropicTokenCounter(BaseTokenCounter):
         if not api_key:
             api_key = os.getenv("AZURE_AI_API_KEY")
 
-        # Get API base from deployment config or environment
-        api_base = litellm_params.get("api_base")
-        if not api_base:
-            api_base = os.getenv("AZURE_AI_API_BASE")
-
-        if not api_key:
-            verbose_logger.warning("No Azure AI API key found for token counting")
-            return None
+        api_base: Final = (
+            litellm_params.get("api_base")
+            or os.getenv("AZURE_AI_API_BASE")
+            or os.getenv("AZURE_API_BASE")
+            or litellm.api_base
+        )
 
         if not api_base:
-            verbose_logger.warning("No Azure AI API base found for token counting")
-            return None
+            return _missing_api_base_response(request_model, model_to_use)
 
         try:
             result: Final = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request(

litellm/llms/azure_ai/anthropic/count_tokens/transformation.py

@@ -24,7 +24,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
 
     def get_required_headers(
         self,
-        api_key: str,
+        api_key: str | None,
         litellm_params: dict[str, object] | None = None,
     ) -> dict[str, str]:
         """
@@ -45,12 +45,12 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
             "Content-Type": "application/json",
             "anthropic-version": "2023-06-01",
             "anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
-            "x-api-key": api_key,  # Azure AI Anthropic requires this header
+            **({"x-api-key": api_key} if api_key else {}),
         }
 
         # Also set up Azure auth headers for flexibility
         litellm_params = litellm_params or {}
-        if "api_key" not in litellm_params:
+        if api_key and "api_key" not in litellm_params:
             litellm_params["api_key"] = api_key
 
         litellm_params_obj: Final = GenericLiteLLMParams.model_validate(litellm_params)
@@ -58,8 +58,9 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
         # Get Azure auth headers (api-key or Authorization)
         azure_headers = BaseAzureLLM.base_validate_azure_environment(headers={}, litellm_params=litellm_params_obj)
 
-        # Merge Azure auth headers
         headers.update(azure_headers)
+        if "api-key" in azure_headers:
+            headers.setdefault("x-api-key", azure_headers["api-key"])
 
         return headers
 

litellm/llms/base_llm/chat/transformation.py

@@ -488,6 +488,10 @@ class BaseConfig(ABC):
         """
         return True
 
+    @property
+    def accepts_reasoning_summary_in_reasoning_effort(self) -> bool:
+        return False
+
     def post_stream_processing(self, stream: object) -> object:
         """Hook for providers to post-process streaming responses. Default: pass-through."""
         return stream

litellm/llms/base_llm/realtime/transcription_protocol.py

@@ -102,6 +102,16 @@ def json_integer(
     return value
 
 
+def uses_server_vad(
+    update: TranscriptionSessionUpdate, backend: str, drop_params: bool, error: ProtocolErrorType
+) -> bool:
+    if update.turn_detection_disabled:
+        return False
+    if update.turn_detection_type in (None, "server_vad") or drop_params:
+        return True
+    raise error(f"{backend} supports server_vad turn detection or null. Set drop_params to use server_vad instead")
+
+
 def new_event_id() -> str:
     return f"event_{uuid.uuid4().hex}"
 

litellm/llms/base_llm/videos/async_task_utils.py

@@ -0,0 +1,123 @@
+import base64
+from collections.abc import Mapping
+from io import BufferedReader, BytesIO
+from itertools import accumulate, repeat
+from pathlib import Path, PurePath
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+from litellm.images.utils import ImageEditRequestUtils
+from litellm.types.videos.utils import decode_video_id_with_provider
+
+_FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object])
+_FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
+
+
+def _fields(value: object) -> Mapping[str, object] | None:
+    try:
+        return _FIELDS_ADAPTER.validate_python(value)
+    except ValidationError:
+        return None
+
+
+def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]:
+    return _fields(getattr(logging_obj, "litellm_params", None)) or {}
+
+
+def _id_safe_name(name: object) -> str | None:
+    return name if isinstance(name, str) and name and ";" not in name else None
+
+
+def _stamped_models(metadata: object) -> tuple[str, str | None] | None:
+    fields: Final = _fields(metadata)
+    if fields is None:
+        return None
+    requested: Final = _id_safe_name(fields.get("model_group"))
+    served: Final = _id_safe_name(fields.get("deployment_model_name"))
+    if requested is None:
+        return None
+    return requested, None if served is None or served == requested or "*" in served else served
+
+
+def routed_video_models(logging_obj: object) -> tuple[str, str | None] | None:
+    """
+    A later status or content request checks the key's model access against the model encoded in the id, and
+    routes by it. One upstream model name can be served by several deployments (regions, accounts), so only the
+    group that served the task can send its polls back to the deployment that created it, while a key may be
+    scoped to the alias, routing group or wildcard name it asked for instead. The id carries both names: the one
+    the router was asked for on this hop, and the group that served it when that differs and is not a wildcard.
+    The router stamps both into litellm_metadata, and metadata carries a copy only when the caller sent none.
+    """
+    params: Final = _logged_litellm_params(logging_obj)
+    stamped: Final = (_stamped_models(params.get(key)) for key in ("litellm_metadata", "metadata"))
+    return next((models for models in stamped if models), None)
+
+
+def polled_video_models(logging_obj: object) -> tuple[str | None, str | None]:
+    """
+    The served group survives into the returned id only when the router sent this poll to it, so an edited id
+    cannot write any name it likes back.
+    """
+    video_id: Final = _logged_litellm_params(logging_obj).get("video_id")
+    if not isinstance(video_id, str):
+        return None, None
+    decoded: Final = decode_video_id_with_provider(video_id)
+    served: Final = decoded.get("served_model")
+    routed: Final = routed_video_models(logging_obj)
+    return decoded.get("model_id") or None, served if routed is None or served in routed else None
+
+
+def _tuple_payload(file_obj: object) -> object:
+    if not isinstance(file_obj, tuple):
+        return file_obj
+    file_tuple: Final = _FILE_TUPLE_ADAPTER.validate_python(file_obj)
+    return file_tuple[1] if len(file_tuple) >= 2 else None
+
+
+def _file_content(file_obj: object) -> object:
+    unwrapped: Final = accumulate(repeat(None), lambda current, _: _tuple_payload(current), initial=file_obj)
+    return next(item for item in unwrapped if not isinstance(item, tuple))
+
+
+def input_reference_bytes(file_obj: object) -> bytes:
+    content: Final = _file_content(file_obj)
+    if isinstance(content, PurePath):
+        return Path(content).read_bytes()
+    if isinstance(content, (BytesIO, BufferedReader)):
+        current_position: Final = content.tell()
+        _ = content.seek(0)
+        data: Final = content.read()
+        _ = content.seek(current_position)
+        return data
+    if isinstance(content, bytes):
+        return content
+    if isinstance(content, bytearray):
+        return bytes(content)
+    read: Final = getattr(content, "read", None)
+    if callable(read):
+        read_data: Final[object] = read()
+        if isinstance(read_data, bytes):
+            return read_data
+    raise ValueError("input_reference must be a URL string, bytes, a path, a file object, or a file tuple")
+
+
+def _reference_object_url(reference: Mapping[str, object]) -> str:
+    image_url: Final = reference.get("image_url")
+    if isinstance(image_url, str) and image_url:
+        return image_url
+    raise ValueError("input_reference as an object must carry an image_url, a file_id is not supported")
+
+
+def input_reference_url(image: object) -> str:
+    """A string is a URL the provider fetches itself and is never opened as a local path. OpenAI's object form
+    carries that URL, or a data URL, in ``image_url``."""
+    if isinstance(image, str):
+        return image
+    reference: Final = _fields(image)
+    if reference is not None:
+        return _reference_object_url(reference)
+    content: Final = input_reference_bytes(image)
+    content_type: Final = ImageEditRequestUtils.get_image_content_type(content)
+    encoded: Final = base64.b64encode(content).decode("utf-8")
+    return f"data:{content_type};base64,{encoded}"

litellm/llms/bedrock/base_aws_llm.py

@@ -99,11 +99,16 @@ def _assume_role_params(
             )
 
 
-_SecureTransportBool = TypedDict("_SecureTransportBool", {"aws:SecureTransport": ReadOnly[Literal["true"]]})
+_SecureTransportBool: Final = TypedDict("_SecureTransportBool", {"aws:SecureTransport": ReadOnly[Literal["true"]]})
+
+_CalledViaLastEquals: Final = TypedDict(
+    "_CalledViaLastEquals", {"aws:CalledViaLast": ReadOnly[Literal["aws-external-anthropic.amazonaws.com"]]}
+)
 
 
-class _SecureTransportCondition(TypedDict):
+class _SessionPolicyCondition(TypedDict):
     Bool: ReadOnly[_SecureTransportBool]
+    StringEquals: NotRequired[ReadOnly[_CalledViaLastEquals]]
 
 
 class _SessionPolicyStatement(TypedDict):
@@ -111,7 +116,7 @@ class _SessionPolicyStatement(TypedDict):
     Effect: ReadOnly[Literal["Allow"]]
     Action: ReadOnly[tuple[str, ...]]
     Resource: ReadOnly[Literal["*"]]
-    Condition: ReadOnly[_SecureTransportCondition]
+    Condition: ReadOnly[_SessionPolicyCondition]
 
 
 class WebIdentitySessionPolicy(TypedDict):
@@ -147,11 +152,24 @@ _WEB_IDENTITY_SESSION_POLICY_ACTIONS: Final[Mapping[str, tuple[str, ...]]] = Map
             "aws-external-anthropic:Get*",
             "aws-external-anthropic:List*",
         ),
+        "ClaudePlatformOutboundIdentityLiteLLM": (
+            "sts:GetWebIdentityToken",
+            "sts:TagGetWebIdentityToken",
+        ),
         "BedrockMantleLiteLLM": ("bedrock-mantle:CreateInference",),
     }
 )
 
-_SECURE_TRANSPORT_ONLY: Final = _SecureTransportCondition(Bool=_SecureTransportBool({"aws:SecureTransport": "true"}))
+_SECURE_TRANSPORT_ONLY: Final = _SessionPolicyCondition(Bool=_SecureTransportBool({"aws:SecureTransport": "true"}))
+
+_WEB_IDENTITY_SESSION_POLICY_CONDITIONS: Final[Mapping[str, _SessionPolicyCondition]] = MappingProxyType(
+    {
+        "ClaudePlatformOutboundIdentityLiteLLM": _SessionPolicyCondition(
+            Bool=_SecureTransportBool({"aws:SecureTransport": "true"}),
+            StringEquals=_CalledViaLastEquals({"aws:CalledViaLast": "aws-external-anthropic.amazonaws.com"}),
+        ),
+    }
+)
 
 
 def build_web_identity_session_policy() -> WebIdentitySessionPolicy:
@@ -163,7 +181,7 @@ def build_web_identity_session_policy() -> WebIdentitySessionPolicy:
                 Effect="Allow",
                 Action=actions,
                 Resource="*",
-                Condition=_SECURE_TRANSPORT_ONLY,
+                Condition=_WEB_IDENTITY_SESSION_POLICY_CONDITIONS.get(sid, _SECURE_TRANSPORT_ONLY),
             )
             for sid, actions in _WEB_IDENTITY_SESSION_POLICY_ACTIONS.items()
         ),

litellm/llms/bedrock/chat/invoke_handler.py

@@ -13,6 +13,7 @@ from litellm.caching.caching import InMemoryCache
 from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
 from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers
 from litellm.litellm_core_utils.optional_imports import ensure_optional_import
 from litellm.litellm_core_utils.prompt_templates.common_utils import native_redacted_thinking_data
 from litellm.llms.anthropic.chat.handler import (
@@ -448,6 +449,7 @@ class _EventStreamTally:
             f"x-amzn-requestid={_response_header(response_headers, 'x-amzn-requestid')!r}, "
             f"{self.bytes_received} bytes received"
         )
+        request_id_headers: Final = provider_request_id_headers(response_headers)
         if not self.events:
             return BedrockError(
                 status_code=502,
@@ -455,6 +457,7 @@ class _EventStreamTally:
                     "Bedrock answered the stream with HTTP 200 but its body decoded to no events "
                     f"({detail}, first bytes={self.head!r})"
                 ),
+                headers=request_id_headers,
             )
         if not self.recognized_events:
             return BedrockError(
@@ -464,10 +467,12 @@ class _EventStreamTally:
                     f"event type (event types={sorted(self.unrecognized_event_types)}, {detail}, "
                     f"first payload={self.unrecognized_head!r})"
                 ),
+                headers=request_id_headers,
             )
         return BedrockError(
             status_code=502,
             message=f"Bedrock stream ended with {undecoded} undecoded bytes after {self.events} events ({detail})",
+            headers=request_id_headers,
         )
 
 
@@ -837,11 +842,12 @@ class AWSEventStreamDecoder:
 
         event_stream_buffer: Final = EventStreamBuffer()
         tally: Final = _EventStreamTally(get_bedrock_stream_event_statuses())
+        request_id_headers: Final = provider_request_id_headers(response_headers)
         for chunk in iterator:
             event_stream_buffer.add_data(chunk)
             tally.add_chunk(chunk)
             for event in event_stream_buffer:
-                message = self._decode_event(event, tally)
+                message = self._decode_event(event, tally, request_id_headers)
                 if message:
                     # sse_event = ServerSentEvent(data=message, event="completion")
                     _data = json.loads(message)
@@ -858,11 +864,12 @@ class AWSEventStreamDecoder:
 
         event_stream_buffer: Final = EventStreamBuffer()
         tally: Final = _EventStreamTally(get_bedrock_stream_event_statuses())
+        request_id_headers: Final = provider_request_id_headers(response_headers)
         async for chunk in iterator:
             event_stream_buffer.add_data(chunk)
             tally.add_chunk(chunk)
             for event in event_stream_buffer:
-                message = self._decode_event(event, tally)
+                message = self._decode_event(event, tally, request_id_headers)
                 if message:
                     _data = json.loads(message)
                     yield self.chunk_parser(chunk_data=_data)
@@ -882,27 +889,34 @@ class AWSEventStreamDecoder:
             )
         return response_stream_shape
 
-    def _decode_event(self, event: "EventStreamMessage", tally: _EventStreamTally) -> str | None:
+    def _decode_event(
+        self, event: "EventStreamMessage", tally: _EventStreamTally, request_id_headers: httpx.Headers | None
+    ) -> str | None:
         response_stream_shape: Final = self._response_stream_shape()
         response_dict: Final = bedrock_event_stream_response(event)
         tally.add_event(event, response_dict["headers"])
-        return self._parse_message_from_response(response_dict, response_stream_shape)
+        return self._parse_message_from_response(response_dict, response_stream_shape, request_id_headers)
 
     def _parse_message_from_event(self, event: "EventStreamMessage") -> str | None:
         response_stream_shape: Final = self._response_stream_shape()
         return self._parse_message_from_response(bedrock_event_stream_response(event), response_stream_shape)
 
     def _parse_message_from_response(
-        self, response_dict: BedrockEventStreamResponseDict, response_stream_shape: "Shape"
+        self,
+        response_dict: BedrockEventStreamResponseDict,
+        response_stream_shape: "Shape",
+        request_id_headers: httpx.Headers | None = None,
     ) -> str | None:
         parsed_response: Final = self.parser.parse(response_dict, response_stream_shape)
 
         if response_dict["status_code"] != 200:
-            raise build_bedrock_stream_error(response_dict, response_stream_shape)
+            raise build_bedrock_stream_error(response_dict, response_stream_shape, headers=request_id_headers)
         event_type: Final = bedrock_event_stream_header(response_dict["headers"], ":event-type")
         event_error_status: Final = bedrock_stream_event_error_status(event_type)
         if event_type is not None and event_error_status is not None:
-            raise build_bedrock_stream_event_error(event_type, event_error_status, response_dict["body"])
+            raise build_bedrock_stream_event_error(
+                event_type, event_error_status, response_dict["body"], headers=request_id_headers
+            )
         if "chunk" in parsed_response:
             chunk = parsed_response.get("chunk")
             if not chunk:

litellm/llms/bedrock/chat/invoke_transformations/base_invoke_transformation.py

@@ -14,9 +14,9 @@ from litellm.litellm_core_utils.get_litellm_params import stored_control_options
 from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
 from litellm.litellm_core_utils.prompt_templates.factory import (
     cohere_message_pt,
-    custom_prompt,
     deepseek_r1_pt,
     prompt_factory,
+    render_custom_prompt_template,
 )
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.bedrock.chat.invoke_handler import make_call, make_sync_call
@@ -601,7 +601,7 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
                 return cast(litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL, provider)
         return None
 
-    def convert_messages_to_prompt(self, model, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
+    def convert_messages_to_prompt(self, model: str, messages, provider, custom_prompt_dict) -> tuple[str, list | None]:
         # handle anthropic prompts and amazon titan prompts
         prompt = ""
         chat_history: list | None = None
@@ -609,11 +609,8 @@ class AmazonInvokeConfig(BaseConfig, BaseAWSLLM):
         if model in custom_prompt_dict:
             # check if the model has a registered custom prompt
             model_prompt_details: Final = custom_prompt_dict[model]
-            prompt = custom_prompt(
-                role_dict=model_prompt_details["roles"],
-                initial_prompt_value=model_prompt_details.get("initial_prompt_value", ""),
-                final_prompt_value=model_prompt_details.get("final_prompt_value", ""),
-                messages=messages,
+            prompt = render_custom_prompt_template(
+                template=model_prompt_details, messages=messages, model=model, llm_provider="bedrock"
             )
             return prompt, None
         ## ELSE

litellm/llms/bedrock/common_utils.py

@@ -1913,13 +1913,18 @@ def bedrock_event_stream_header(headers: Mapping[str, object], name: str) -> str
     return value if isinstance(value, str) else None
 
 
-def build_bedrock_stream_event_error(event_type: str, status_code: int, body: bytes) -> BedrockError:
-    return BedrockError(status_code=status_code, message=f"{event_type} {body.decode(errors='replace')}")
+def build_bedrock_stream_event_error(
+    event_type: str, status_code: int, body: bytes, headers: httpx.Headers | None = None
+) -> BedrockError:
+    return BedrockError(
+        status_code=status_code, message=f"{event_type} {body.decode(errors='replace')}", headers=headers
+    )
 
 
 def build_bedrock_stream_error(
     response_dict: BedrockEventStreamResponseDict,
     response_stream_shape: Shape | None,
+    headers: httpx.Headers | None = None,
 ) -> BedrockError:
     """Build a BedrockError for a non-200 event-stream error event.
 
@@ -1929,12 +1934,14 @@ def build_bedrock_stream_error(
     """
     exception_type: Final = bedrock_event_stream_header(response_dict["headers"], ":exception-type")
     if exception_type is None:
-        return BedrockError(status_code=response_dict["status_code"], message=response_dict["body"].decode())
+        return BedrockError(
+            status_code=response_dict["status_code"], message=response_dict["body"].decode(), headers=headers
+        )
 
     member: Final = _structure_members(response_stream_shape).get(exception_type)
     modeled_status: Final = None if member is None else _modeled_error_status(member)
     status_code: Final = response_dict["status_code"] if modeled_status is None else modeled_status
-    return build_bedrock_stream_event_error(exception_type, status_code, response_dict["body"])
+    return build_bedrock_stream_event_error(exception_type, status_code, response_dict["body"], headers=headers)
 
 
 class BedrockEventStreamDecoderBase:

litellm/llms/codestral/completion/handler.py

@@ -13,8 +13,8 @@ import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
 from litellm.litellm_core_utils.prompt_templates.factory import (
-    custom_prompt,
     prompt_factory,
+    render_custom_prompt_template,
 )
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -261,11 +261,8 @@ class CodestralTextCompletion:
         if model in custom_prompt_dict:
             # check if the model has a registered custom prompt
             model_prompt_details: Final = custom_prompt_dict[model]
-            prompt = custom_prompt(
-                role_dict=model_prompt_details["roles"],
-                initial_prompt_value=model_prompt_details["initial_prompt_value"],
-                final_prompt_value=model_prompt_details["final_prompt_value"],
-                messages=messages,
+            prompt = render_custom_prompt_template(
+                template=model_prompt_details, messages=messages, model=model, llm_provider="codestral"
             )
         else:
             prompt = prompt_factory(model=model, messages=messages)

litellm/llms/dashscope/videos/transformation.py

@@ -3,13 +3,10 @@ DashScope (Alibaba Model Studio) async video task API: create, poll and download
 delete or remix endpoint, so those raise.
 """
 
-import base64
 import json
 from collections.abc import Mapping
 from datetime import datetime
-from io import BufferedReader, BytesIO
 from math import log
-from pathlib import Path, PurePath
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Annotated, Final
 
@@ -19,9 +16,13 @@ from pydantic import BaseModel, Field, StrictBool, StrictInt, StrictStr, TypeAda
 
 import litellm
 from litellm.exceptions import UnsupportedParamsError
-from litellm.images.utils import ImageEditRequestUtils
 from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.videos.async_task_utils import (
+    input_reference_url,
+    polled_video_models,
+    routed_video_models,
+)
 from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
 from litellm.llms.custom_httpx.http_handler import (
     _get_httpx_client,  # pyright: ignore[reportPrivateUsage, reportUnknownVariableType]  # house cached-client factory has no public alias and its stub leaves params untyped
@@ -32,7 +33,6 @@ from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import ModelInfo
 from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject
 from litellm.types.videos.utils import (
-    decode_video_id_with_provider,
     encode_video_id_with_provider,
     extract_original_video_id,
 )
@@ -86,7 +86,6 @@ _EMPTY_PARAMS: Final[dict[str, object]] = {}  # mutable-ok: BaseVideoConfig cont
 
 _TASK_RESPONSE_ADAPTER: Final = TypeAdapter(_TaskResponse)
 _PARAMETERS_ADAPTER: Final = TypeAdapter(dict[str, object])
-_FILE_TUPLE_ADAPTER: Final = TypeAdapter(tuple[object, ...])
 
 _STATUS_MAP: Final = MappingProxyType(
     {
@@ -383,39 +382,6 @@ def _is_priced_tier(resolution: object, priced_tiers: tuple[str, ...]) -> bool:
     return isinstance(resolution, str) and resolution.strip().upper() in priced_tiers
 
 
-def _read_all_bytes(file_obj: object) -> bytes:
-    if isinstance(file_obj, tuple):
-        file_tuple: Final = _FILE_TUPLE_ADAPTER.validate_python(file_obj)
-        return _read_all_bytes(file_tuple[1] if len(file_tuple) >= 2 else None)
-    if isinstance(file_obj, PurePath):
-        return Path(file_obj).read_bytes()
-    if isinstance(file_obj, (BytesIO, BufferedReader)):
-        current_position: Final = file_obj.tell()
-        file_obj.seek(0)
-        content: Final = file_obj.read()
-        file_obj.seek(current_position)
-        return content
-    if isinstance(file_obj, bytes):
-        return file_obj
-    if isinstance(file_obj, bytearray):
-        return bytes(file_obj)
-    read: Final = getattr(file_obj, "read", None)
-    if callable(read):
-        data: Final = read()
-        if isinstance(data, bytes):
-            return data
-    raise ValueError("input_reference must be a URL string, bytes, a path, a file object, or a file tuple")
-
-
-def _image_url(image: object) -> str:
-    if isinstance(image, str):
-        return image
-    content: Final = _read_all_bytes(image)
-    content_type: Final = ImageEditRequestUtils.get_image_content_type(content)
-    encoded: Final = base64.b64encode(content).decode("utf-8")
-    return f"data:{content_type};base64,{encoded}"
-
-
 def _media_reference_type(model: str) -> str | None:
     """Models without the field predate the media array and take the image on the flat ``img_url`` field."""
     info: Final = _dashscope_model_info(model)
@@ -558,58 +524,6 @@ def _video_url_from_task(task: _TaskResponse, raw_response: httpx.Response) -> s
     raise _video_error(raw_response, 502, "Video URL not found in task response. The task may not have succeeded yet.")
 
 
-def _logged_litellm_params(logging_obj: object) -> Mapping[str, object]:
-    litellm_params: Final[object] = getattr(logging_obj, "litellm_params", None)
-    try:
-        return _PARAMETERS_ADAPTER.validate_python(litellm_params)
-    except ValidationError:
-        return _EMPTY_PARAMS
-
-
-def _polled_models(logging_obj: object) -> tuple[str | None, str | None]:
-    """
-    The proxy routes by the models encoded in the polled id, so they must survive into the returned id. The served
-    group survives only when the router sent this poll to it, so an edited id cannot write any name it likes back.
-    """
-    video_id: Final = _logged_litellm_params(logging_obj).get("video_id")
-    if not isinstance(video_id, str):
-        return None, None
-    decoded: Final = decode_video_id_with_provider(video_id)
-    served: Final = decoded.get("served_model")
-    routed: Final = _poll_models(logging_obj)
-    return decoded.get("model_id") or None, served if routed is None or served in routed else None
-
-
-def _poll_models(logging_obj: object) -> tuple[str, str | None] | None:
-    """
-    A later status or content request checks the key's model access against the model encoded in the id, and
-    routes by it. DashScope serves the same model name from several regional hosts, so only the group that
-    served the task can send its polls back to the region that created it, while a key may be scoped to the
-    alias, routing group or wildcard name it asked for instead. The id carries both names: the one the router
-    was asked for on this hop, and the group that served it when that differs and is not a wildcard. The router
-    stamps both into litellm_metadata, and metadata carries a copy only when the caller sent none.
-    """
-    params: Final = _logged_litellm_params(logging_obj)
-    stamped: Final = (_stamped_poll_models(params.get(key)) for key in ("litellm_metadata", "metadata"))
-    return next((models for models in stamped if models), None)
-
-
-def _stamped_poll_models(metadata: object) -> tuple[str, str | None] | None:
-    try:
-        fields: Final = _PARAMETERS_ADAPTER.validate_python(metadata)
-    except ValidationError:
-        return None
-    requested: Final = _id_safe_name(fields.get("model_group"))
-    served: Final = _id_safe_name(fields.get("deployment_model_name"))
-    if requested is None:
-        return None
-    return requested, None if served is None or served == requested or "*" in served else served
-
-
-def _id_safe_name(name: object) -> str | None:
-    return name if isinstance(name, str) and name and ";" not in name else None
-
-
 class DashScopeVideoConfig(BaseVideoConfig):
     def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: BaseVideoConfig contract returns list
         return [
@@ -804,7 +718,7 @@ class DashScopeVideoConfig(BaseVideoConfig):
         input_reference: Final = video_create_optional_request_params.get("input_reference")
         if input_reference is None:
             return None, None
-        image_url: Final = _image_url(input_reference)
+        image_url: Final = input_reference_url(input_reference)
         media_type: Final = _media_reference_type(model)
         if media_type is None:
             return "img_url", image_url
@@ -823,7 +737,7 @@ class DashScopeVideoConfig(BaseVideoConfig):
         requested: Final = self._requested_parameters(request_data)
         video_obj: Final = _video_object_from_task(task, raw_response, model=model, requested=requested)
         if custom_llm_provider and video_obj.id:
-            requested_model, served_model = _poll_models(logging_obj) or (model, None)
+            requested_model, served_model = routed_video_models(logging_obj) or (model, None)
             video_obj.id = encode_video_id_with_provider(
                 video_obj.id, custom_llm_provider, requested_model, served_model
             )
@@ -869,7 +783,7 @@ class DashScopeVideoConfig(BaseVideoConfig):
         client: "HTTPHandler | None" = None,
     ) -> VideoObject:
         task: Final = _parse_task_response(raw_response)
-        model, served_model = _polled_models(logging_obj)
+        model, served_model = polled_video_models(logging_obj)
         video_obj: Final = _video_object_from_task(task, raw_response, model=model)
         if custom_llm_provider and video_obj.id:
             video_obj.id = encode_video_id_with_provider(video_obj.id, custom_llm_provider, model, served_model)

litellm/llms/databricks/chat/transformation.py

@@ -20,6 +20,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
     merge_consecutive_system_messages,
     strip_litellm_internal_message_fields,
     strip_name_from_message,
+    strip_prefill_prefix_field,
 )
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.base_utils import (
@@ -487,6 +488,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
         Databricks does not support:
         - 'name' in user message.
         - litellm's internal `thinking_blocks` / `reasoning_content` on assistant messages.
+        - the `prefix` prefill flag Mistral and DeepSeek read, which Claude does not need to continue an assistant turn.
         - non-Claude json_object "json" checks read only string content, so a single text block is sent as a string.
         """
         new_messages = []
@@ -496,7 +498,7 @@ class DatabricksConfig(DatabricksBase, OpenAILikeChatConfig, AnthropicConfig):
             else:
                 _message = message
             _message = strip_name_from_message(_message, allowed_name_roles=["user"])
-            _message = strip_litellm_internal_message_fields(_message)
+            _message = strip_prefill_prefix_field(strip_litellm_internal_message_fields(_message))
             # Move message-level cache_control into a content block when content is a string.
             if "cache_control" in _message and isinstance(_message.get("content"), str):
                 _message = self._move_cache_control_into_string_content_block(_message)

litellm/llms/gemini/realtime/transformation.py

@@ -161,6 +161,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
         """Google AI Studio Gemini 3.5+ accepts ``id`` on functionResponses; Vertex AI rejects it."""
         return True
 
+    def _rejects_vad_tuning_with_detection_off(self) -> bool:
+        """Google AI Studio closes the socket (1007) on padding or silence tuning next to ``disabled: true``."""
+        return True
+
     @staticmethod
     def _usage_detail_alias(details: Mapping[str, int | None] | None, defaults: dict[str, int]) -> dict[str, int]:
         if not isinstance(details, dict):
@@ -279,10 +283,15 @@ class GeminiRealtimeConfig(BaseRealtimeConfig):
 
         OpenAI ``semantic_vad`` has no Gemini Live equivalent — return an empty
         dict so callers omit ``realtimeInputConfig`` (mapping it with
-        ``disabled: true`` breaks native-audio sessions).
+        ``disabled: true`` breaks native-audio sessions). An explicit
+        ``create_response: false`` maps to detection off alone where the backend
+        rejects tuning next to ``disabled: true``.
         """
         if isinstance(value, dict) and value.get("type") == "semantic_vad" and "create_response" not in value:
             return AutomaticActivityDetection()
+        auto_response_off: Final = "create_response" in value and value["create_response"] is False
+        if auto_response_off and self._rejects_vad_tuning_with_detection_off():
+            return AutomaticActivityDetection(disabled=True)
 
         automatic_activity_dection: Final = AutomaticActivityDetection()
         if "create_response" in value and isinstance(value["create_response"], bool):

litellm/llms/meta/realtime/transformation.py

@@ -26,6 +26,7 @@ from litellm.llms.base_llm.realtime.transcription_protocol import (
     speech_event,
     transcription_session,
     transcription_session_created_event,
+    uses_server_vad,
 )
 from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig
 from litellm.secret_managers.main import get_secret_str
@@ -230,15 +231,7 @@ def _parse_sample_rate(update: TranscriptionSessionUpdate) -> MuseSampleRate:
     return 16_000 if rate == 16_000 else 24_000
 
 
-def _parse_mode(update: TranscriptionSessionUpdate) -> MuseMode:
-    if update.turn_detection_disabled:
-        return "PUSH_TO_TALK"
-    if update.turn_detection_type not in (None, "server_vad"):
-        raise MuseProtocolError("Muse Voice supports server_vad turn detection or null")
-    return "ENDPOINTING"
-
-
-def parse_session_update(payload: str, expected_model: str) -> MuseSessionConfig:
+def parse_session_update(payload: str, expected_model: str, *, drop_params: bool = False) -> MuseSessionConfig:
     update: Final = parse_transcription_session_update(payload, MuseProtocolError)
     if update.session_type not in (None, "transcription", "realtime"):
         raise MuseProtocolError("Muse Voice supports transcription sessions only")
@@ -253,7 +246,7 @@ def parse_session_update(payload: str, expected_model: str) -> MuseSessionConfig
         raise MuseProtocolError("realtime session model cannot be changed")
     return MuseSessionConfig(
         model=normalized_model,
-        mode=_parse_mode(update),
+        mode="ENDPOINTING" if uses_server_vad(update, "Muse Voice", drop_params, MuseProtocolError) else "PUSH_TO_TALK",
         sample_rate=_parse_sample_rate(update),
         language_bias=() if update.language is None else (normalize_language(update.language),),
     )
@@ -441,9 +434,11 @@ class MetaRealtimeConfig(BaseRealtimeConfig):
         *,
         monotonic: Callable[[], float] = time.monotonic,
         sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+        drop_params: bool = False,
     ) -> None:
         self._monotonic: Final = monotonic
         self._sleep: Final = sleep
+        self._drop_params: Final = drop_params
         self._transformer: Final = MuseEventTransformer()
         self._access_token: str | None = None
         self._config: MuseSessionConfig | None = None
@@ -553,7 +548,7 @@ class MetaRealtimeConfig(BaseRealtimeConfig):
         access_token: Final = self._access_token
         if access_token is None:
             raise MuseProtocolError("Meta API key was not validated before the session was configured")
-        config: Final = parse_session_update(message, model)
+        config: Final = parse_session_update(message, model, drop_params=self._drop_params)
         self._config = config
         self._transformer.configure(config)
         return (json.dumps(config.handshake(access_token), separators=(",", ":")),)

litellm/llms/ollama/completion/transformation.py

@@ -16,8 +16,8 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
     convert_to_ollama_image,
-    custom_prompt,
     ollama_pt,
+    render_custom_prompt_template,
 )
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
     async_inline_remote_media,
@@ -437,11 +437,8 @@ class OllamaConfig(BaseConfig):
         if model in custom_prompt_dict:
             # check if the model has a registered custom prompt
             model_prompt_details: Final = custom_prompt_dict[model]
-            ollama_prompt = custom_prompt(
-                role_dict=model_prompt_details["roles"],
-                initial_prompt_value=model_prompt_details["initial_prompt_value"],
-                final_prompt_value=model_prompt_details["final_prompt_value"],
-                messages=messages,
+            ollama_prompt = render_custom_prompt_template(
+                template=model_prompt_details, messages=messages, model=model, llm_provider="ollama"
             )
         elif text_completion_request:  # handle `/completions` requests
             ollama_prompt = get_str_from_messages(messages=messages)

litellm/llms/openai/responses/count_tokens/token_counter.py

@@ -2,20 +2,21 @@
 OpenAI Token Counter implementation using the Responses API /input_tokens endpoint.
 """
 
-import os
 from collections.abc import Mapping, Sequence
 from typing import Any, Final
 
+import openai
 from pydantic import JsonValue
 from typing_extensions import assert_never
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.llms.anthropic.pass_through.convertible_blocks import unconvertible_blocks_error
 from litellm.llms.anthropic.pass_through.responses_adapters.transformation import (
     LiteLLMAnthropicToResponsesAPIAdapter,
 )
 from litellm.llms.base_llm.base_utils import BaseTokenCounter
-from litellm.llms.openai.common_utils import OpenAIError
+from litellm.llms.openai.common_utils import OpenAIError, get_openai_credentials
 from litellm.llms.openai.responses.count_tokens.handler import (
     OpenAICountTokensHandler,
 )
@@ -23,12 +24,26 @@ from litellm.llms.openai.responses.count_tokens.transformation import (
     OpenAICountTokensConfig,
     ResponsesCountRequestParts,
 )
+from litellm.llms.openai.workload_identity import (
+    OpenAIWorkloadIdentityConfig,
+    get_workload_identity_bearer_token,
+    resolve_openai_workload_identity_config,
+)
 from litellm.types.utils import LlmProviders, TokenCountMessagesFormat, TokenCountResponse
 
 # Global handler instance - reuse across all token counting requests
 openai_count_tokens_handler: Final = OpenAICountTokensHandler()
 
 
+def _workload_identity_config(litellm_params: Mapping[str, object]) -> OpenAIWorkloadIdentityConfig | None:
+    api_base: Final = litellm_params.get("api_base")
+    return resolve_openai_workload_identity_config(
+        api_key=None,
+        api_base=api_base if isinstance(api_base, str) else None,
+        litellm_params=litellm_params,
+    )
+
+
 class OpenAITokenCounter(BaseTokenCounter):
     """Token counter implementation for OpenAI provider using the Responses API."""
 
@@ -107,16 +122,27 @@ class OpenAITokenCounter(BaseTokenCounter):
         deployment = deployment or {}
         litellm_params: Final = deployment.get("litellm_params", {})
 
-        # Get OpenAI API key from deployment config or environment
-        api_key = litellm_params.get("api_key")
-        if not api_key:
-            api_key = os.getenv("OPENAI_API_KEY")
+        credentials: Final = get_openai_credentials(
+            api_base=litellm_params.get("api_base"), api_key=litellm_params.get("api_key")
+        )
+        credential: Final = credentials.api_key or _workload_identity_config(litellm_params)
 
-        if not api_key:
-            verbose_logger.warning("No OpenAI API key found for token counting")
-            return None
+        if credential is None:
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="openai_api",
+                error=True,
+                error_message=(
+                    f"No OpenAI API key is set for model {request_model or model_to_use}, so tokens can't be "
+                    "counted. Set api_key or workload identity on the deployment, or OPENAI_API_KEY in the proxy "
+                    "environment."
+                ),
+                status_code=401,
+            )
 
-        api_base: Final = litellm_params.get("api_base")
+        api_base: Final = credentials.api_base
 
         input_items, instructions, counted_tools, text = self._responses_request_parts(
             model_to_use=model_to_use,
@@ -132,6 +158,11 @@ class OpenAITokenCounter(BaseTokenCounter):
             return None
 
         try:
+            api_key: Final = (
+                credential
+                if isinstance(credential, str)
+                else await asyncify(get_workload_identity_bearer_token)(credential)
+            )
             result: Final = await openai_count_tokens_handler.handle_count_tokens_request(
                 model=model_to_use,
                 input=input_items,
@@ -163,6 +194,17 @@ class OpenAITokenCounter(BaseTokenCounter):
                 error_message=e.message,
                 status_code=e.status_code,
             )
+        except openai.AuthenticationError as e:
+            verbose_logger.warning("OpenAI workload identity token exchange failed: %s", e.message)
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="openai_api",
+                error=True,
+                error_message=e.message,
+                status_code=401,
+            )
         except Exception as e:
             verbose_logger.warning("Error calling OpenAI CountTokens API: %s", e)
             return TokenCountResponse(

litellm/llms/petals/completion/handler.py

@@ -5,8 +5,8 @@ from typing import Final
 
 import litellm
 from litellm.litellm_core_utils.prompt_templates.factory import (
-    custom_prompt,
     prompt_factory,
+    render_custom_prompt_template,
 )
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
@@ -43,11 +43,8 @@ def completion(
     if model in litellm.custom_prompt_dict:
         # check if the model has a registered custom prompt
         model_prompt_details: Final = litellm.custom_prompt_dict[model]
-        prompt = custom_prompt(
-            role_dict=model_prompt_details["roles"],
-            initial_prompt_value=model_prompt_details["initial_prompt_value"],
-            final_prompt_value=model_prompt_details["final_prompt_value"],
-            messages=messages,
+        prompt = render_custom_prompt_template(
+            template=model_prompt_details, messages=messages, model=model, llm_provider="petals"
         )
     else:
         prompt = prompt_factory(model=model, messages=messages)

litellm/llms/predibase/chat/transformation.py

@@ -10,8 +10,8 @@ import litellm
 from litellm.constants import DEFAULT_MAX_TOKENS
 from litellm.litellm_core_utils.core_helpers import map_finish_reason
 from litellm.litellm_core_utils.prompt_templates.factory import (
-    custom_prompt,
     prompt_factory,
+    render_custom_prompt_template,
 )
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.types.llms.openai import AllMessageValues
@@ -277,11 +277,8 @@ class PredibaseConfig(BaseConfig):
         custom_prompt_dict: Final = litellm_params.get("custom_prompt_dict", {})
         if model in custom_prompt_dict:
             model_prompt_details: Final = custom_prompt_dict[model]
-            prompt = custom_prompt(
-                role_dict=model_prompt_details["roles"],
-                initial_prompt_value=model_prompt_details["initial_prompt_value"],
-                final_prompt_value=model_prompt_details["final_prompt_value"],
-                messages=messages,
+            prompt = render_custom_prompt_template(
+                template=model_prompt_details, messages=messages, model=model, llm_provider="predibase"
             )
         else:
             prompt = prompt_factory(model=model, messages=messages)

litellm/llms/sagemaker/completion/transformation.py

@@ -15,8 +15,8 @@ from httpx._models import Headers, Response
 import litellm
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.litellm_core_utils.prompt_templates.factory import (
-    custom_prompt,
     prompt_factory,
+    render_custom_prompt_template,
 )
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.types.llms.openai import AllMessageValues
@@ -122,20 +122,14 @@ class SagemakerConfig(BaseConfig):
         if model in custom_prompt_dict:
             # check if the model has a registered custom prompt
             model_prompt_details = custom_prompt_dict[model]
-            prompt = custom_prompt(
-                role_dict=model_prompt_details.get("roles", None),
-                initial_prompt_value=model_prompt_details.get("initial_prompt_value", ""),
-                final_prompt_value=model_prompt_details.get("final_prompt_value", ""),
-                messages=messages,
+            prompt = render_custom_prompt_template(
+                template=model_prompt_details, messages=messages, model=model, llm_provider="sagemaker"
             )
         elif hf_model_name in custom_prompt_dict:
             # check if the base huggingface model has a registered custom prompt
             model_prompt_details = custom_prompt_dict[hf_model_name]
-            prompt = custom_prompt(
-                role_dict=model_prompt_details.get("roles", None),
-                initial_prompt_value=model_prompt_details.get("initial_prompt_value", ""),
-                final_prompt_value=model_prompt_details.get("final_prompt_value", ""),
-                messages=messages,
+            prompt = render_custom_prompt_template(
+                template=model_prompt_details, messages=messages, model=model, llm_provider="sagemaker"
             )
         else:
             if hf_model_name is None:

litellm/llms/soniox/audio_transcription/handler.py

@@ -19,11 +19,14 @@ import asyncio
 import math
 import time
 from collections.abc import Coroutine, Mapping, Sequence
+from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final
 
 import httpx
+from pydantic import TypeAdapter
 from typing_extensions import ReadOnly, TypedDict
 
+import litellm
 from litellm.litellm_core_utils.audio_utils.utils import (
     get_audio_file_name,
     process_audio_file,
@@ -35,7 +38,12 @@ from litellm.llms.custom_httpx.http_handler import (
     get_httpx_client,
 )
 from litellm.llms.soniox.audio_transcription.transformation import (
+    SONIOX_HANDLER_ONLY_PARAMS,
     SonioxAudioTranscriptionConfig,
+    SonioxInvalidFormParam,
+    decode_soniox_form_params,
+    raise_soniox_form_error,
+    soniox_language_hints,
 )
 from litellm.llms.soniox.common_utils import (
     SONIOX_DEFAULT_CLEANUP,
@@ -58,6 +66,39 @@ else:
     LiteLLMLoggingObj = Any
 
 
+_CLEANUP_ENTRIES: Final = TypeAdapter(tuple[object, ...])
+_NOT_FORWARDED_TO_SONIOX: Final[frozenset[str]] = frozenset(
+    (*SONIOX_HANDLER_ONLY_PARAMS, "audio_url", "file_id", "response_format", "language")
+)
+
+
+def _optional_str(value: object) -> str | None:
+    return None if value is None else str(value)
+
+
+def _float_or_default(value: object, default: float) -> float:
+    try:
+        parsed: Final = float(str(value))
+    except ValueError:
+        return default
+    return parsed if math.isfinite(parsed) else default
+
+
+def _int_or_default(value: object, default: int) -> int:
+    try:
+        return int(value) if isinstance(value, (int, float)) else int(str(value))
+    except (ValueError, OverflowError):
+        return default
+
+
+def _cleanup_targets(raw: object) -> tuple[str, ...]:
+    if raw is None:
+        return ()
+    if isinstance(raw, str):
+        return (raw,)
+    return tuple(target for target in _CLEANUP_ENTRIES.validate_python(raw) if isinstance(target, str))
+
+
 class _TranscriptionMeta(TypedDict, total=False):
     """Fields the handler reads from a Soniox transcription object."""
 
@@ -182,7 +223,7 @@ class SonioxAudioTranscriptionHandler:
     ) -> tuple[
         dict[str, str],  # auth headers
         str,  # api_base (no trailing slash)
-        dict[str, object],  # body for POST /v1/transcriptions (without file_id/audio_url)
+        Mapping[str, object],  # body for POST /v1/transcriptions (without file_id/audio_url)
         _HandlerOptions,  # handler-only options (poll interval, cleanup, ...)
     ]:
         # Validate env -> auth headers.
@@ -198,52 +239,42 @@ class SonioxAudioTranscriptionHandler:
 
         base_url: Final = get_soniox_api_base(api_base)
 
-        # Operate on a local copy so we don't mutate the caller's dict
-        # (the caller may reuse `optional_params` for retries or logging).
-        params: Final = dict(optional_params)
-
-        # Pull handler-only kwargs out of params so they aren't sent
-        # to Soniox.
-        poll_interval = float(params.pop("soniox_polling_interval", SONIOX_DEFAULT_POLL_INTERVAL))
-        try:
-            max_attempts = int(params.pop("soniox_max_polling_attempts", SONIOX_DEFAULT_MAX_POLL_ATTEMPTS))
-        except (ValueError, OverflowError):
-            max_attempts = SONIOX_DEFAULT_MAX_POLL_ATTEMPTS
-        cleanup_raw: Final = params.pop("soniox_cleanup", SONIOX_DEFAULT_CLEANUP)
-        if cleanup_raw is None:
-            cleanup: list[str] = []
-        elif isinstance(cleanup_raw, str):
-            cleanup = [cleanup_raw]
-        else:
-            cleanup = list(cleanup_raw)
-        filename_override: Final = params.pop("filename", None)
+        decoded: Final = decode_soniox_form_params(
+            optional_params, drop_params=litellm_params.get("drop_params") is True or litellm.drop_params is True
+        )
+        if isinstance(decoded, SonioxInvalidFormParam):
+            raise_soniox_form_error(decoded)
 
         # Server-side clamps. Caller-supplied poll settings (from request kwargs)
         # are bounded so an authenticated caller cannot force a worker into a
         # tight poll loop (zero interval) or pin it indefinitely (huge attempt
         # count). Total polling time is bounded by
         #   SONIOX_MAX_POLL_ATTEMPTS * SONIOX_MAX_POLL_INTERVAL.
-        if not math.isfinite(poll_interval):
-            poll_interval = SONIOX_DEFAULT_POLL_INTERVAL
-        clamped_poll_interval: Final = max(SONIOX_MIN_POLL_INTERVAL, min(poll_interval, SONIOX_MAX_POLL_INTERVAL))
-        clamped_max_attempts: Final = max(1, min(max_attempts, SONIOX_MAX_POLL_ATTEMPTS))
+        poll_interval: Final = _float_or_default(decoded.get("soniox_polling_interval"), SONIOX_DEFAULT_POLL_INTERVAL)
+        max_attempts: Final = _int_or_default(
+            decoded.get("soniox_max_polling_attempts"), SONIOX_DEFAULT_MAX_POLL_ATTEMPTS
+        )
+        cleanup: Final = _cleanup_targets(decoded.get("soniox_cleanup", SONIOX_DEFAULT_CLEANUP))
+        language_hints: Final = soniox_language_hints(decoded.get("language"), decoded.get("language_hints"))
 
         # response_format is handled by LiteLLM post-processing, not Soniox.
         handler_opts: Final[_HandlerOptions] = {
-            "poll_interval": clamped_poll_interval,
-            "max_attempts": clamped_max_attempts,
+            "poll_interval": max(SONIOX_MIN_POLL_INTERVAL, min(poll_interval, SONIOX_MAX_POLL_INTERVAL)),
+            "max_attempts": max(1, min(max_attempts, SONIOX_MAX_POLL_ATTEMPTS)),
             "cleanup": cleanup,
-            "filename_override": filename_override,
-            "audio_url": params.pop("audio_url", None),
-            "file_id": params.pop("file_id", None),
-            "response_format": params.pop("response_format", None),
+            "filename_override": _optional_str(decoded.get("filename")),
+            "audio_url": _optional_str(decoded.get("audio_url")),
+            "file_id": _optional_str(decoded.get("file_id")),
+            "response_format": _optional_str(decoded.get("response_format")),
         }
 
-        # Soniox does not accept `language` directly; map_openai_params should
-        # already have translated it, but drop any leftover to be safe.
-        params.pop("language", None)
-
-        return auth_headers, base_url, params, handler_opts
+        provider_params: Final = MappingProxyType(
+            {
+                **{key: value for key, value in decoded.items() if key not in _NOT_FORWARDED_TO_SONIOX},
+                **({} if language_hints is None else {"language_hints": language_hints}),
+            }
+        )
+        return auth_headers, base_url, provider_params, handler_opts
 
     def _build_create_body(
         self,

litellm/llms/soniox/audio_transcription/transformation.py

@@ -9,9 +9,14 @@ async API requires multiple HTTP calls and does not fit the single-request
 contract of `base_llm_http_handler.audio_transcriptions`.
 """
 
-from typing import Any, Final
+import re
+from collections.abc import Mapping
+from dataclasses import dataclass
+from types import MappingProxyType
+from typing import Any, Final, NoReturn
 
 from httpx import Headers, Response
+from pydantic import TypeAdapter, ValidationError
 
 from litellm.llms.base_llm.audio_transcription.transformation import (
     AudioTranscriptionRequestData,
@@ -26,6 +31,7 @@ from litellm.llms.soniox.common_utils import (
     render_soniox_tokens_as_srt,
     render_soniox_tokens_as_vtt,
 )
+from litellm.llms.soniox.types import SonioxContext
 from litellm.types.llms.openai import (
     AllMessageValues,
     OpenAIAudioTranscriptionOptionalParams,
@@ -58,6 +64,152 @@ SONIOX_HANDLER_ONLY_PARAMS: Final[list[str]] = [
 ]
 
 
+SONIOX_BOOL_PARAMS: Final[frozenset[str]] = frozenset(
+    {"enable_speaker_diarization", "enable_language_identification", "language_hints_strict"}
+)
+_JSON_OBJECT: Final = TypeAdapter(dict[str, object])
+_STRUCTURED_CONTEXT: Final = TypeAdapter(SonioxContext)
+_FORM_BOOL: Final = TypeAdapter(bool)
+_ITEMS: Final = TypeAdapter(tuple[object, ...])
+_BRACKETED_FORM_FIELD: Final = re.compile(
+    r"(?P<param>context|translation)\[(?P<field>\w+)\](?:\[\]\[(?P<entry_field>\w+)\])?"
+)
+_BracketedField = tuple[str, str, str | None, object]
+
+
+@dataclass(frozen=True, slots=True)
+class SonioxInvalidFormParam:
+    key: str
+    value: str
+    expected: str
+
+
+def raise_soniox_form_error(error: SonioxInvalidFormParam) -> NoReturn:
+    raise SonioxException(
+        message=f"`{error.key}` must be {error.expected}, got {error.value!r}", status_code=400, headers=None
+    )
+
+
+def soniox_language_hints(language: object, hints: object) -> object:
+    """Soniox has no `language` field, so the OpenAI `language` becomes `language_hints` when none are set."""
+    if hints is not None or not isinstance(language, str) or not language:
+        return hints
+    return (language,)
+
+
+def _decode_bool(key: str, value: str) -> bool | SonioxInvalidFormParam:
+    try:
+        return _FORM_BOOL.validate_python(value.strip())
+    except ValidationError:
+        return SonioxInvalidFormParam(key=key, value=value, expected="a boolean")
+
+
+def _decode_polling_interval(value: str) -> str | SonioxInvalidFormParam:
+    try:
+        float(value)
+    except ValueError:
+        return SonioxInvalidFormParam(key="soniox_polling_interval", value=value, expected="a number")
+    return value
+
+
+def _decode_context(value: str) -> SonioxContext | str:
+    if not value.lstrip().startswith("{"):
+        return value
+    try:
+        return _STRUCTURED_CONTEXT.validate_json(value)
+    except ValidationError:
+        return value
+
+
+def _decode_translation(value: str) -> Mapping[str, object] | SonioxInvalidFormParam:
+    try:
+        return _JSON_OBJECT.validate_json(value)
+    except ValidationError:
+        return SonioxInvalidFormParam(key="translation", value=value, expected="a JSON object")
+
+
+def _decode_language_hints(value: str) -> tuple[object, ...] | SonioxInvalidFormParam:
+    if not value.lstrip().startswith("["):
+        return tuple(hint.strip() for hint in value.split(",") if hint.strip())
+    try:
+        return _ITEMS.validate_json(value)
+    except ValidationError:
+        return SonioxInvalidFormParam(key="language_hints", value=value, expected="a JSON list")
+
+
+def _decode_form_value(key: str, value: object) -> object | SonioxInvalidFormParam:
+    if not isinstance(value, str):
+        return value
+    if key in SONIOX_BOOL_PARAMS:
+        return _decode_bool(key, value)
+    if key == "soniox_polling_interval":
+        return _decode_polling_interval(value)
+    if key == "context":
+        return _decode_context(value)
+    if key == "translation":
+        return _decode_translation(value)
+    if key == "language_hints":
+        return _decode_language_hints(value)
+    return value
+
+
+def _nested_form_param(param: str, fields: tuple[_BracketedField, ...]) -> Mapping[str, object]:
+    owned: Final = tuple(field for field in fields if field[0] == param)
+    flat: Final = {name: value for _, name, entry_field, value in owned if entry_field is None}
+    listed: Final = {name for _, name, entry_field, _ in owned if entry_field is not None}
+    return {
+        **flat,
+        **{
+            name: (
+                {
+                    entry_field: value
+                    for _, owner, entry_field, value in owned
+                    if owner == name and entry_field is not None
+                },
+            )
+            for name in listed
+        },
+    }
+
+
+def _is_soniox_context(value: object) -> bool:
+    try:
+        _STRUCTURED_CONTEXT.validate_python(value)
+    except ValidationError:
+        return False
+    return True
+
+
+def _nest_bracketed_form_fields(params: Mapping[str, object]) -> Mapping[str, object]:
+    """The OpenAI SDK sends a dict in `extra_body` as `context[terms]` and `context[general][][key]` form keys."""
+    matches: Final = {key: _BRACKETED_FORM_FIELD.fullmatch(key) for key in params}
+    fields: Final = tuple(
+        (match["param"], match["field"], match["entry_field"], params[key]) for key, match in matches.items() if match
+    )
+    nested: Final = {param: _nested_form_param(param, fields) for param in {field[0] for field in fields}}
+    kept: Final = {param: value for param, value in nested.items() if param != "context" or _is_soniox_context(value)}
+    unbracketed: Final = {
+        key: value for key, value in params.items() if (match := matches[key]) is None or match["param"] not in kept
+    }
+    return {**kept, **unbracketed}
+
+
+def decode_soniox_form_params(
+    optional_params: Mapping[str, object],
+    drop_params: bool = False,
+) -> Mapping[str, object] | SonioxInvalidFormParam:
+    """Multipart form fields reach the proxy as strings; restore the JSON types Soniox expects."""
+    decoded: Final = {
+        key: _decode_form_value(key, value) for key, value in _nest_bracketed_form_fields(optional_params).items()
+    }
+    invalid: Final = next((value for value in decoded.values() if isinstance(value, SonioxInvalidFormParam)), None)
+    if invalid is not None and not drop_params:
+        return invalid
+    return MappingProxyType(
+        {key: value for key, value in decoded.items() if not isinstance(value, SonioxInvalidFormParam)}
+    )
+
+
 class SonioxAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
     """Configuration for Soniox async speech-to-text transcription."""
 
@@ -74,25 +226,26 @@ class SonioxAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
         model: str,
         drop_params: bool,
     ) -> dict:
-        # Translate the OpenAI `language` param into Soniox `language_hints`.
-        if "language" in non_default_params and non_default_params["language"]:
-            language: Final = non_default_params["language"]
-            existing_hints: Final = optional_params.get("language_hints")
-            if not existing_hints:
-                optional_params["language_hints"] = [language]
-            elif language not in existing_hints:
-                optional_params["language_hints"] = [language] + list(existing_hints)
-
-        # Capture response_format for post-processing (not sent to Soniox API).
-        if "response_format" in non_default_params:
-            optional_params["response_format"] = non_default_params["response_format"]
-
-        # Pass through Soniox-native kwargs unchanged.
-        for key in SONIOX_PASSTHROUGH_PARAMS + SONIOX_HANDLER_ONLY_PARAMS:
-            if key in non_default_params and non_default_params[key] is not None:
-                optional_params[key] = non_default_params[key]
-
-        return optional_params
+        language: Final[object] = non_default_params.get("language")
+        language_params: Final[Mapping[str, object]] = (
+            {
+                "language": language,
+                "language_hints": soniox_language_hints(language, optional_params.get("language_hints")),
+            }
+            if language
+            else {}
+        )
+        response_format_params: Final[Mapping[str, object]] = (
+            {"response_format": non_default_params["response_format"]}
+            if "response_format" in non_default_params
+            else {}
+        )
+        passthrough_params: Final[Mapping[str, object]] = {
+            key: non_default_params[key]
+            for key in SONIOX_PASSTHROUGH_PARAMS + SONIOX_HANDLER_ONLY_PARAMS
+            if non_default_params.get(key) is not None
+        }
+        return {**optional_params, **language_params, **response_format_params, **passthrough_params}
 
     def get_error_class(self, error_message: str, status_code: int, headers: dict | Headers) -> BaseLLMException:
         return SonioxException(message=error_message, status_code=status_code, headers=headers)

litellm/llms/soniox/types.py

@@ -0,0 +1,24 @@
+from collections.abc import Sequence
+
+from pydantic import ConfigDict, with_config
+from typing_extensions import ReadOnly, TypedDict
+
+
+class SonioxContextGeneralEntry(TypedDict):
+    key: ReadOnly[str]
+    value: ReadOnly[str]
+
+
+class SonioxTranslationTerm(TypedDict):
+    source: ReadOnly[str]
+    target: ReadOnly[str]
+
+
+@with_config(ConfigDict(extra="forbid"))
+class SonioxContext(TypedDict, total=False):
+    """Soniox `context` request field: https://soniox.com/docs/stt/concepts/context"""
+
+    general: ReadOnly[Sequence[SonioxContextGeneralEntry]]
+    text: ReadOnly[str]
+    terms: ReadOnly[Sequence[str]]
+    translation_terms: ReadOnly[Sequence[SonioxTranslationTerm]]

litellm/llms/vertex_ai/audio_transcription/realtime_transformation.py

@@ -13,7 +13,6 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.llms.base_llm.realtime.transcription_protocol import (
     RealtimeTranscriptionProtocolError,
     TranscriptionAudioFormat,
-    TranscriptionSessionUpdate,
     completed_event,
     decode_pcm16_append,
     delta_event,
@@ -23,6 +22,7 @@ from litellm.llms.base_llm.realtime.transcription_protocol import (
     speech_event,
     transcription_session,
     transcription_session_created_event,
+    uses_server_vad,
 )
 from litellm.llms.base_llm.realtime.transformation import BaseRealtimeConfig, RealtimeBackend
 from litellm.llms.vertex_ai.audio_transcription.transformation import (
@@ -125,7 +125,7 @@ def default_session_config(model: str) -> ChirpSessionConfig:
     )
 
 
-def parse_chirp_session_update(payload: str, expected_model: str) -> ChirpSessionConfig:
+def parse_chirp_session_update(payload: str, expected_model: str, *, drop_params: bool = False) -> ChirpSessionConfig:
     update: Final = parse_transcription_session_update(payload, ChirpProtocolError)
     if update.session_type not in (None, "transcription", "realtime"):
         raise ChirpProtocolError("Speech-to-Text streaming supports transcription sessions only")
@@ -141,7 +141,7 @@ def parse_chirp_session_update(payload: str, expected_model: str) -> ChirpSessio
         model=model,
         language=None if update.language is None else normalize_transcription_language_to_bcp47(update.language),
         sample_rate=_parse_sample_rate(update.audio_format),
-        server_vad=_parse_server_vad(update),
+        server_vad=uses_server_vad(update, "Speech-to-Text streaming", drop_params, ChirpProtocolError),
     )
 
 
@@ -161,14 +161,6 @@ def _parse_sample_rate(audio_format: TranscriptionAudioFormat | None) -> int:
     return rate
 
 
-def _parse_server_vad(update: TranscriptionSessionUpdate) -> bool:
-    if update.turn_detection_disabled:
-        return False
-    if update.turn_detection_type not in (None, "server_vad"):
-        raise ChirpProtocolError("Speech-to-Text streaming supports server_vad turn detection or null")
-    return True
-
-
 def session_created_event(config: ChirpSessionConfig, session_id: str) -> OpenAIRealtimeTranscriptionSessionCreated:
     return transcription_session_created_event(config.openai_session(session_id))
 
@@ -331,11 +323,13 @@ class VertexChirpRealtimeConfig(BaseRealtimeConfig):
         project: str,
         location: str | None,
         backend_factory: Callable[[SpeechStreamingTarget], RealtimeBackend] = _default_backend_factory,
+        drop_params: bool = False,
     ) -> None:
         self._resolve_access_token: Final = resolve_access_token
         self._project: Final = validate_vertex_transcription_project_id(project)
         self._location: Final = validate_vertex_transcription_location(location, DEFAULT_SPEECH_TO_TEXT_LOCATION)
         self._backend_factory: Final = backend_factory
+        self._drop_params: Final = drop_params
         self._transformer: Final = ChirpEventTransformer()
         self._config: ChirpSessionConfig | None = None
         self._session_id: str | None = None
@@ -423,7 +417,7 @@ class VertexChirpRealtimeConfig(BaseRealtimeConfig):
         if self._config is not None:
             verbose_logger.debug("Speech-to-Text streaming: ignoring session.update after the stream was configured")
             return ()
-        config: Final = parse_chirp_session_update(message, model)
+        config: Final = parse_chirp_session_update(message, model, drop_params=self._drop_params)
         self._config = config
         self._transformer.configure(config, self._session_id or f"sess_{uuid.uuid4().hex}")
         return (config.configure_command(),)

litellm/llms/vertex_ai/common_utils.py

@@ -738,14 +738,14 @@ def build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
     # refs recursively and correctly detects/skips circular references.
     unpack_defs(parameters, defs)
 
+    _convert_schema_types(parameters)
+
     # 5. Nullable fields:
     #     * https://github.com/pydantic/pydantic/issues/1270
     #     * https://stackoverflow.com/a/58841311
     #     * https://github.com/pydantic/pydantic/discussions/4872
     convert_anyof_null_to_nullable(parameters)
 
-    _convert_schema_types(parameters)
-
     # Handle empty strings in enum values - Gemini doesn't accept empty strings in enums
     _fix_enum_empty_strings(parameters)
 
@@ -926,34 +926,28 @@ def convert_anyof_null_to_nullable(schema, depth=0):
         )
     """ Converts null objects within anyOf by removing them and adding nullable to all remaining objects """
     anyof: Final = schema.get("anyOf", None)
-    if anyof is not None:
-        contains_null = False
+    if isinstance(anyof, list):
+        non_null_branches: Final = [
+            atype for atype in anyof if not (isinstance(atype, dict) and atype.get("type") == "null")
+        ]
+        contains_null: Final = len(non_null_branches) < len(anyof)
+        anyof[:] = non_null_branches or anyof[:1]
         for atype in anyof:
-            if isinstance(atype, dict) and atype.get("type") == "null":
-                # remove null type
-                anyof.remove(atype)
-                contains_null = True
-            elif "type" not in atype and len(atype) == 0:
-                # Handle empty object case
+            if not isinstance(atype, dict):
+                continue
+            if not atype:
                 atype["type"] = "object"
-
-        if len(anyof) == 0:
-            # Edge case: response schema with only null type present is invalid in Vertex AI
-            raise ValueError(
-                "Invalid input: AnyOf schema with only null type is not supported. Please provide a non-null type."
-            )
-
-        if contains_null:
-            # set all types to nullable following guidance found here: https://cloud.google.com/vertex-ai/generative-ai/docs/samples/generativeaionvertexai-gemini-controlled-generation-response-schema-3#generativeaionvertexai_gemini_controlled_generation_response_schema_3-python
-            # Empty `items: {}` on array branches is left in place; downstream
-            # process_items() converts it to {"type": "object"}, which Vertex
-            # requires whenever type == "array" (even inside anyOf).
-            for atype in anyof:
+            if contains_null:
+                # set all types to nullable following guidance found here: https://cloud.google.com/vertex-ai/generative-ai/docs/samples/generativeaionvertexai-gemini-controlled-generation-response-schema-3#generativeaionvertexai_gemini_controlled_generation_response_schema_3-python
+                # Empty `items: {}` on array branches is left in place; downstream
+                # process_items() converts it to {"type": "object"}, which Vertex
+                # requires whenever type == "array" (even inside anyOf).
                 atype["nullable"] = True
+            convert_anyof_null_to_nullable(atype, depth=depth + 1)
 
     properties: Final = schema.get("properties", None)
     if properties is not None:
-        for name, value in properties.items():
+        for value in properties.values():
             convert_anyof_null_to_nullable(value, depth=depth + 1)
 
     items: Final = schema.get("items", None)

litellm/llms/vertex_ai/gemini/transformation.py

@@ -1154,6 +1154,7 @@ def transform_request_body(
             user_response_schema_message: Final = response_schema_prompt(
                 model=model,
                 response_schema=optional_params.get("response_schema"),
+                llm_provider=custom_llm_provider,
             )
             messages.append({"role": "user", "content": user_response_schema_message})
             optional_params.pop("response_schema")

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -27,6 +27,7 @@ from litellm.constants import (
     DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET_GEMINI_2_5_PRO,
 )
 from litellm.exceptions import UnsupportedParamsError
+from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 from litellm.litellm_core_utils.json_fragment_accumulator import JSONFragmentAccumulator
 from litellm.litellm_core_utils.prompt_templates.factory import (
     encode_tool_call_id_with_signature,
@@ -926,11 +927,38 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 # - Requires propertyOrdering
                 optional_params["response_schema"] = self._map_response_schema(value=schema)
 
+    @property
+    def accepts_reasoning_summary_in_reasoning_effort(self) -> bool:
+        return True
+
+    @staticmethod
+    def _include_thoughts(requested: bool | None) -> bool:
+        if requested is not None:
+            return requested
+        return normalize_drop_params(litellm.gemini_include_thoughts_default) is not False
+
+    @staticmethod
+    def _include_thoughts_requested_by_reasoning_effort(reasoning_effort: object) -> bool | None:
+        match reasoning_effort:
+            case {"summary": str()}:
+                return True
+            case _:
+                return None
+
+    @staticmethod
+    def _map_reasoning_effort(effort: str, model: str, reasoning_effort: object) -> GeminiThinkingConfig:
+        include_thoughts: Final = VertexGeminiConfig._include_thoughts_requested_by_reasoning_effort(reasoning_effort)
+        if VertexGeminiConfig._is_gemini_3_or_newer(model):
+            return VertexGeminiConfig._map_reasoning_effort_to_thinking_level(effort, model, include_thoughts)
+        return VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(effort, model, include_thoughts)
+
     @staticmethod
     def _map_reasoning_effort_to_thinking_budget(
         reasoning_effort: str,
         model: str | None = None,
+        include_thoughts: bool | None = None,
     ) -> GeminiThinkingConfig:
+        sends_thoughts: Final = VertexGeminiConfig._include_thoughts(include_thoughts)
         if reasoning_effort == "minimal":
             # Use model-specific minimum thinking budget or fallback
             # Check for exact matches first, then partial matches
@@ -945,22 +973,22 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
 
             return {
                 "thinkingBudget": budget,
-                "includeThoughts": True,
+                "includeThoughts": sends_thoughts,
             }
         elif reasoning_effort == "low":
             return {
                 "thinkingBudget": DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET,
-                "includeThoughts": True,
+                "includeThoughts": sends_thoughts,
             }
         elif reasoning_effort == "medium":
             return {
                 "thinkingBudget": DEFAULT_REASONING_EFFORT_MEDIUM_THINKING_BUDGET,
-                "includeThoughts": True,
+                "includeThoughts": sends_thoughts,
             }
         elif reasoning_effort == "high":
             return {
                 "thinkingBudget": DEFAULT_REASONING_EFFORT_HIGH_THINKING_BUDGET,
-                "includeThoughts": True,
+                "includeThoughts": sends_thoughts,
             }
         elif reasoning_effort == "disable":
             return {
@@ -987,6 +1015,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
     def _map_reasoning_effort_to_thinking_level(
         reasoning_effort: str,
         model: str | None = None,
+        include_thoughts: bool | None = None,
     ) -> GeminiThinkingConfig:
         """
         Map reasoning_effort to thinking_level for Gemini 3+ models.
@@ -998,17 +1027,18 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             GeminiThinkingConfig with thinkingLevel and includeThoughts
         """
         supports_minimal: Final = bool(model) and VertexGeminiConfig._supports_minimal_thinking_level(model)
+        sends_thoughts: Final = VertexGeminiConfig._include_thoughts(include_thoughts)
         if reasoning_effort == "minimal":
             if supports_minimal:
-                return {"thinkingLevel": "minimal", "includeThoughts": True}
+                return {"thinkingLevel": "minimal", "includeThoughts": sends_thoughts}
             else:
-                return {"thinkingLevel": "low", "includeThoughts": True}
+                return {"thinkingLevel": "low", "includeThoughts": sends_thoughts}
         elif reasoning_effort == "low":
-            return {"thinkingLevel": "low", "includeThoughts": True}
+            return {"thinkingLevel": "low", "includeThoughts": sends_thoughts}
         elif reasoning_effort == "medium":
-            return {"thinkingLevel": "medium", "includeThoughts": True}
+            return {"thinkingLevel": "medium", "includeThoughts": sends_thoughts}
         elif reasoning_effort == "high":
-            return {"thinkingLevel": "high", "includeThoughts": True}
+            return {"thinkingLevel": "high", "includeThoughts": sends_thoughts}
         elif reasoning_effort in ("disable", "none"):
             return {
                 "thinkingLevel": "minimal" if supports_minimal else "low",
@@ -1061,6 +1091,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                     status_code=400,
                 )
 
+    @staticmethod
+    def _thinking_asks_for_thoughts(thinking_param: Mapping[str, object]) -> bool:
+        display: Final = thinking_param.get("display")
+        return isinstance(thinking_param.get("summary"), str) or (display is not None and display != "omitted")
+
     @staticmethod
     def _map_thinking_param(
         thinking_param: AnthropicThinkingParam,
@@ -1068,6 +1103,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
     ) -> GeminiThinkingConfig:
         thinking_enabled: Final = thinking_param.get("type") == "enabled"
         thinking_budget: Final = thinking_param.get("budget_tokens")
+        asks_for_thoughts: Final = VertexGeminiConfig._thinking_asks_for_thoughts(thinking_param)
+        sends_thoughts: Final = VertexGeminiConfig._include_thoughts(True if asks_for_thoughts else None)
 
         params: Final[GeminiThinkingConfig] = {}
 
@@ -1077,7 +1114,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 if thinking_budget == 0:
                     params["includeThoughts"] = False
                 else:
-                    params["includeThoughts"] = True
+                    params["includeThoughts"] = sends_thoughts
                     # Follow provider defaults unless explicitly opted into legacy behavior.
                     if litellm.enable_gemini_default_thinking_level_low is True:
                         params["thinkingLevel"] = (
@@ -1089,7 +1126,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
         else:
             # For older Gemini models, use thinkingBudget
             if thinking_enabled and not VertexGeminiConfig._is_thinking_budget_zero(thinking_budget):
-                params["includeThoughts"] = True
+                params["includeThoughts"] = sends_thoughts
             if thinking_budget is not None and isinstance(thinking_budget, int):
                 params["thinkingBudget"] = thinking_budget
 
@@ -1304,14 +1341,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                         param_name="reasoning_effort",
                         param_description="thinking_budget",
                     )
-                    if VertexGeminiConfig._is_gemini_3_or_newer(model):
-                        optional_params["thinkingConfig"] = VertexGeminiConfig._map_reasoning_effort_to_thinking_level(
-                            effort_value, model
-                        )
-                    else:
-                        optional_params["thinkingConfig"] = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(
-                            effort_value, model
-                        )
+                    optional_params["thinkingConfig"] = VertexGeminiConfig._map_reasoning_effort(
+                        effort=effort_value, model=model, reasoning_effort=value
+                    )
             elif param == "thinking":
                 # Validate no conflict with thinking_level
                 VertexGeminiConfig._validate_thinking_config_conflicts(

litellm/llms/vertex_ai/realtime/transformation.py

@@ -47,6 +47,9 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig):
     def _include_function_response_id(self) -> bool:
         return False
 
+    def _rejects_vad_tuning_with_detection_off(self) -> bool:
+        return False
+
     # ------------------------------------------------------------------
     # URL
     # ------------------------------------------------------------------
@@ -248,9 +251,12 @@ def vertex_realtime_config(
     resolve_access_token: Callable[[], Awaitable[str]],
     project: str,
     location: str | None,
+    drop_params: bool,
 ) -> VertexAIRealtimeConfig | VertexChirpRealtimeConfig:
     if is_vertex_speech_to_text_model(model):
-        return VertexChirpRealtimeConfig(resolve_access_token=resolve_access_token, project=project, location=location)
+        return VertexChirpRealtimeConfig(
+            resolve_access_token=resolve_access_token, project=project, location=location, drop_params=drop_params
+        )
     return VertexAIRealtimeConfig(
         access_token=access_token,
         project=project,

litellm/llms/vllm/completion/handler.py

@@ -5,8 +5,8 @@ from typing import Final
 import httpx
 
 from litellm.litellm_core_utils.prompt_templates.factory import (
-    custom_prompt,
     prompt_factory,
+    render_custom_prompt_template,
 )
 from litellm.utils import ModelResponse, Usage
 
@@ -56,11 +56,8 @@ def completion(
     if model in custom_prompt_dict:
         # check if the model has a registered custom prompt
         model_prompt_details: Final = custom_prompt_dict[model]
-        prompt = custom_prompt(
-            role_dict=model_prompt_details["roles"],
-            initial_prompt_value=model_prompt_details["initial_prompt_value"],
-            final_prompt_value=model_prompt_details["final_prompt_value"],
-            messages=messages,
+        prompt = render_custom_prompt_template(
+            template=model_prompt_details, messages=messages, model=model, llm_provider="vllm"
         )
     else:
         prompt = prompt_factory(model=model, messages=messages)
@@ -144,11 +141,8 @@ def batch_completions(model: str, messages: list, optional_params=None, custom_p
         # check if the model has a registered custom prompt
         model_prompt_details: Final = custom_prompt_dict[model]
         for message in messages:
-            prompt = custom_prompt(
-                role_dict=model_prompt_details["roles"],
-                initial_prompt_value=model_prompt_details["initial_prompt_value"],
-                final_prompt_value=model_prompt_details["final_prompt_value"],
-                messages=message,
+            prompt = render_custom_prompt_template(
+                template=model_prompt_details, messages=message, model=model, llm_provider="vllm"
             )
             prompts.append(prompt)
     else:

litellm/llms/volcengine/videos/transformation.py

@@ -0,0 +1,904 @@
+import json
+from collections.abc import Mapping, Sequence
+from math import log
+from types import MappingProxyType
+from typing import TYPE_CHECKING, Final, TypeVar
+
+import httpx
+from httpx._types import RequestFiles
+from pydantic import BaseModel, StrictBool, StrictInt, StrictStr, TypeAdapter, ValidationError
+
+import litellm
+from litellm.exceptions import UnsupportedParamsError
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.llms.base_llm.videos.async_task_utils import (
+    input_reference_url,
+    polled_video_models,
+    routed_video_models,
+)
+from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
+from litellm.llms.custom_httpx.http_handler import (
+    AsyncHTTPHandler,
+    HTTPHandler,
+    _get_httpx_client,  # pyright: ignore[reportPrivateUsage, reportUnknownVariableType]  # house cached-client factory has no public alias and its stub leaves params untyped
+    get_async_httpx_client,  # pyright: ignore[reportUnknownVariableType]  # factory stub leaves params untyped
+)
+from litellm.llms.volcengine.common_utils import get_volcengine_base_url
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import ModelInfo
+from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject
+from litellm.types.videos.utils import encode_video_id_with_provider, extract_original_video_id
+from litellm.utils import get_model_info
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging
+
+
+_ParsedT: Final = TypeVar("_ParsedT")
+
+
+class VolcEngineVideoError(BaseLLMException):
+    pass
+
+
+class _TaskError(BaseModel, frozen=True):
+    code: str | None = None
+    message: str | None = None
+
+
+class _TaskContent(BaseModel, frozen=True):
+    video_url: str | None = None
+
+
+class _TaskUsage(BaseModel, frozen=True):
+    total_tokens: int | None = None
+
+
+class _Task(BaseModel, frozen=True):
+    id: str
+    status: str
+    model: str | None = None
+    error: _TaskError | None = None
+    content: _TaskContent | None = None
+    usage: _TaskUsage | None = None
+    created_at: int | None = None
+    updated_at: int | None = None
+    resolution: str | None = None
+    ratio: str | None = None
+    duration: int | float | None = None
+
+
+class _TaskCreateResponse(BaseModel, frozen=True):
+    id: str | None = None
+    error: _TaskError | None = None
+
+
+class _TaskList(BaseModel, frozen=True):
+    items: Sequence[_Task] = ()
+    total: int | None = None
+
+
+class _ContentItem(BaseModel, frozen=True):
+    type: str | None = None
+    role: str | None = None
+
+
+class _KnownFields(BaseModel, frozen=True):
+    resolution: StrictStr | None = None
+    ratio: StrictStr | None = None
+    seed: StrictInt | None = None
+    camera_fixed: StrictBool | None = None
+    watermark: StrictBool | None = None
+    generate_audio: StrictBool | None = None
+
+
+VOLCENGINE_VIDEO_TASKS_PATH: Final = "/api/v3/contents/generations/tasks"
+VOLCENGINE_VIDEO_DEFAULT_DURATION_SECONDS: Final = 4
+VOLCENGINE_VIDEO_DEFAULT_RESOLUTION: Final = "720p"
+_VOLCENGINE_API_MARKER: Final = "/api/"
+_VOLCENGINE_COMPAT_SUFFIXES: Final = ("/api", "/v1")
+_FRAMES_FIELD: Final = "volcengine_video_frames"
+
+_EMPTY_PARAMS: Final[dict[str, object]] = {}  # mutable-ok: BaseVideoConfig contract empty params
+
+_TASK_ADAPTER: Final = TypeAdapter(_Task)
+_TASK_CREATE_ADAPTER: Final = TypeAdapter(_TaskCreateResponse)
+_TASK_LIST_ADAPTER: Final = TypeAdapter(_TaskList)
+_FIELDS_ADAPTER: Final = TypeAdapter(dict[str, object])
+_CONTENT_ITEMS_ADAPTER: Final = TypeAdapter(tuple[dict[str, object], ...])
+_FRAMES_ADAPTER: Final = TypeAdapter(dict[str, dict[str, str]])
+_EMPTY_FRAMES: Final[Mapping[str, Mapping[str, str]]] = MappingProxyType({})
+
+_STATUS_MAP: Final = MappingProxyType(
+    {
+        "queued": "queued",
+        "running": "in_progress",
+        "succeeded": "completed",
+        "failed": "failed",
+        "cancelled": "cancelled",
+        "expired": "failed",
+    }
+)
+_PENDING_STATUSES: Final = frozenset({"queued", "running"})
+_TERMINAL_STATUSES: Final = frozenset({"completed", "failed", "cancelled"})
+
+_CREATE_PASSTHROUGH_KEYS: Final = (
+    "resolution",
+    "ratio",
+    "duration",
+    "seed",
+    "camera_fixed",
+    "watermark",
+    "generate_audio",
+    "service_tier",
+    "priority",
+    "output_format",
+    "execution_expires_after",
+    "callback_url",
+    "safety_identifier",
+    "tools",
+    "omni_reference_task_type",
+)
+
+_REQUEST_METADATA: Final = frozenset({"model", "prompt", "user", "extra_headers", "extra_query", "extra_body"})
+_UNSUPPORTED_BY_VOLCENGINE: Final = frozenset({"characters", "image"})
+_MAPPED_ELSEWHERE: Final = frozenset({*_REQUEST_METADATA, *_UNSUPPORTED_BY_VOLCENGINE, "seconds", "size", "parameters"})
+_READ_FROM_THE_REQUEST: Final = _MAPPED_ELSEWHERE - _UNSUPPORTED_BY_VOLCENGINE
+_LITELLM_CONTROLS: Final = frozenset({"drop_params", "additional_drop_params", "organization"})
+_DROP_KEYS_ADAPTER: Final = TypeAdapter(frozenset[str])
+
+# litellm/videos/utils.py merges the caller's raw ``extra_body`` back over the mapped params, which returns the keys
+# this adapter maps itself a second time, as fields of their own. They are already accounted for, so they count as
+# handled rather than unknown.
+_MAPPED_EXTRA_BODY_COPIES: Final = frozenset({"size", "seconds", "user"})
+_CREATE_HANDLED_KEYS: Final = frozenset(
+    {
+        *_CREATE_PASSTHROUGH_KEYS,
+        "content",
+        "input_reference",
+        "frames",
+        *_MAPPED_EXTRA_BODY_COPIES,
+        *_LITELLM_CONTROLS,
+    }
+)
+
+_VIDEO_INPUT_UNSUPPORTED: Final = (
+    "{operation} is not supported through litellm: Ark bills a task with a video input at a different rate per "
+    "token, and litellm bills the video when the task is created, before that cost is known"
+)
+_INPUT_VIDEO_ROLE: Final = "reference_video"
+_INPUT_VIDEO_TYPE: Final = "video_url"
+
+_RATIO_SET: Final = ("16:9", "9:16", "4:3", "3:4", "1:1", "21:9")
+_RESOLUTION_SHORTEST_SIDE: Final = MappingProxyType({"480p": 480, "720p": 720, "1080p": 1080, "4k": 2160})
+_TIER_FRAME_ASPECT: Final = 16 / 9
+
+
+def _video_error(raw_response: httpx.Response, status_code: int, message: str) -> VolcEngineVideoError:
+    """The provider's own error status wins, and a body that failed on a 2xx answers the caller's status instead,
+    since a 2xx error has the proxy retry a call the provider already rejected."""
+    return VolcEngineVideoError(
+        status_code=raw_response.status_code if raw_response.status_code >= 400 else status_code,
+        message=message,
+        headers=raw_response.headers,
+    )
+
+
+def _parse(adapter: TypeAdapter[_ParsedT], raw_response: httpx.Response) -> _ParsedT:
+    try:
+        return adapter.validate_python(raw_response.json())
+    except (json.JSONDecodeError, ValidationError) as e:
+        raise _video_error(raw_response, 502, f"Unexpected Volcengine video response: {e}") from e
+
+
+def _unsupported(model: str, message: str) -> UnsupportedParamsError:
+    return UnsupportedParamsError(message=message, model=model, llm_provider="volcengine")
+
+
+def _droppable(model: str, message: str) -> UnsupportedParamsError:
+    return _unsupported(model, f"{message}, set drop_params=True to ignore it")
+
+
+def _normalized_api_base(api_base: str) -> str:
+    """The Ark credential is shared with the chat, responses and Anthropic surfaces, so its api_base may carry any of
+    their paths, and the video paths carry their own version segment."""
+    trimmed: Final = api_base.rstrip("/")
+    marker_index: Final = trimmed.rfind(_VOLCENGINE_API_MARKER)
+    if marker_index != -1:
+        return trimmed[:marker_index]
+    suffix: Final = next((suffix for suffix in _VOLCENGINE_COMPAT_SUFFIXES if trimmed.endswith(suffix)), None)
+    return trimmed[: -len(suffix)] if suffix is not None else trimmed
+
+
+def _size_dimensions(size: str) -> tuple[int, int] | None:
+    width_str, separator, height_str = size.partition("x")
+    if not separator or not (width_str.isdigit() and height_str.isdigit()):
+        return None
+    if not int(width_str) or not int(height_str):
+        return None
+    return int(width_str), int(height_str)
+
+
+def _ratio_value(ratio: str) -> float:
+    width_str, _, height_str = ratio.partition(":")
+    return int(width_str) / int(height_str)
+
+
+def _ratio_from_size(size: str) -> str | None:
+    """Ark only takes its named ratios, and its own tiers do not reduce onto them (854x480 is 427:240), so the
+    closest ratio wins."""
+    dimensions: Final = _size_dimensions(size)
+    if dimensions is None:
+        return None
+    width, height = dimensions
+    return min(_RATIO_SET, key=lambda candidate: abs(log(width / height / _ratio_value(candidate))))
+
+
+def _resolution_from_size(size: str) -> str | None:
+    """Ark keeps a tier's pixel count across its ratios (960x960 and 1280x720 are both 720p), so the tier whose 16:9
+    frame has the closest pixel count wins."""
+    dimensions: Final = _size_dimensions(size)
+    if dimensions is None:
+        return None
+    width, height = dimensions
+    return min(
+        _RESOLUTION_SHORTEST_SIDE,
+        key=lambda tier: abs(log(width * height / (_RESOLUTION_SHORTEST_SIDE[tier] ** 2 * _TIER_FRAME_ASPECT))),
+    )
+
+
+def _duration_param(seconds: object) -> int | None:
+    if isinstance(seconds, bool):
+        return None
+    if isinstance(seconds, int):
+        return seconds
+    if not isinstance(seconds, (str, float)):
+        return None
+    try:
+        parsed: Final = float(seconds)
+    except ValueError:
+        return None
+    return int(parsed) if parsed.is_integer() else None
+
+
+def _duration_fields(seconds: object, model: str, drop_params: bool) -> Mapping[str, object]:
+    if seconds is None:
+        return _EMPTY_PARAMS
+    duration: Final = _duration_param(seconds)
+    if duration is not None:
+        return MappingProxyType({"duration": duration})
+    if drop_params:
+        return _EMPTY_PARAMS
+    raise _droppable(model, f"seconds must be a whole number of seconds, got {seconds!r}")
+
+
+def _size_fields(size: object, model: str, drop_params: bool) -> Mapping[str, object]:
+    if size is None:
+        return _EMPTY_PARAMS
+    ratio: Final = _ratio_from_size(size) if isinstance(size, str) else None
+    resolution: Final = _resolution_from_size(size) if isinstance(size, str) else None
+    if ratio is not None and resolution is not None:
+        return MappingProxyType({"ratio": ratio, "resolution": resolution})
+    if drop_params:
+        return _EMPTY_PARAMS
+    raise _droppable(model, f"size must be WIDTHxHEIGHT, got {size!r}")
+
+
+def _parameters_fields(parameters: object, model: str, drop_params: bool) -> Mapping[str, object]:
+    if parameters is None:
+        return _EMPTY_PARAMS
+    try:
+        fields: Final = _FIELDS_ADAPTER.validate_python(parameters)
+    except ValidationError:
+        if drop_params:
+            return _EMPTY_PARAMS
+        raise _droppable(
+            model, f"parameters must be an object of Ark request fields, got {type(parameters).__name__}"
+        ) from None
+    misplaced: Final = tuple(sorted(key for key in fields if key in _READ_FROM_THE_REQUEST))
+    if misplaced and not drop_params:
+        raise _droppable(model, f"send {', '.join(misplaced)} as top-level request fields, not inside parameters")
+    return MappingProxyType({key: value for key, value in fields.items() if key not in _READ_FROM_THE_REQUEST})
+
+
+def _user_fields(user: object, model: str, drop_params: bool) -> Mapping[str, object]:
+    if isinstance(user, str) and user:
+        return MappingProxyType({"safety_identifier": user})
+    if user is None or isinstance(user, str) or drop_params:
+        return _EMPTY_PARAMS
+    raise _droppable(model, f"user must be a string, got {type(user).__name__}")
+
+
+def _additional_drop_keys(params: Mapping[str, object]) -> frozenset[str]:
+    try:
+        return _DROP_KEYS_ADAPTER.validate_python(params.get("additional_drop_params") or ())
+    except ValidationError:
+        return frozenset()
+
+
+def _without(params: Mapping[str, object], keys: frozenset[str]) -> Mapping[str, object]:
+    return MappingProxyType({key: value for key, value in params.items() if key not in keys})
+
+
+def _malformed_fields(fields: Mapping[str, object]) -> tuple[tuple[str, str], ...]:
+    try:
+        _ = _KnownFields.model_validate(dict(fields))
+    except ValidationError as e:
+        return tuple((str(error["loc"][0]), error["msg"]) for error in e.errors() if error["loc"])
+    return ()
+
+
+def _drops_params(litellm_params: GenericLiteLLMParams) -> bool:
+    return litellm_params.drop_params is True or litellm.drop_params is True
+
+
+def _model_info(model: str) -> ModelInfo | None:
+    try:
+        return get_model_info(model=model, custom_llm_provider="volcengine")
+    except Exception:  # noqa: BLE001  # get_model_info raises bare Exception for unmapped models
+        return None
+
+
+def _pricing_model(model: str, litellm_params: GenericLiteLLMParams) -> str:
+    """A deployment priced through ``model_info.base_model``, like an ``ep-`` endpoint id, bills at that model's
+    tiers."""
+    model_info: Final = _FIELDS_ADAPTER.validate_python(litellm_params["model_info"] or _EMPTY_PARAMS)
+    base_model: Final = model_info.get("base_model")
+    return base_model if isinstance(base_model, str) and base_model else model
+
+
+def _priced_tiers(model: str) -> tuple[str, ...] | None:
+    """A tier the model has no price for is one Ark does not offer it at, and litellm would bill it at the base rate.
+    A model with no tier price listed, missing from the cost map or registered unpriced by the router, has nothing
+    to check against."""
+    info: Final = _model_info(model)
+    if info is None:
+        return None
+    return (
+        tuple(tier for tier in _RESOLUTION_SHORTEST_SIDE if info.get(f"output_cost_per_second_{tier}") is not None)
+        or None
+    )
+
+
+def _delivered_frames(model: str | None) -> Mapping[str, Mapping[str, str]]:
+    info: Final = _model_info(model) if model else None
+    provider_specific: Final[object] = info.get("provider_specific_entry") if info is not None else None
+    frames: Final[object] = provider_specific.get(_FRAMES_FIELD) if isinstance(provider_specific, Mapping) else None
+    try:
+        return _FRAMES_ADAPTER.validate_python(frames) if frames is not None else _EMPTY_FRAMES
+    except ValidationError:
+        return _EMPTY_FRAMES
+
+
+def _size_from_geometry(resolution: str | None, ratio: str | None, model: str | None) -> str | None:
+    """Ark echoes a tier and a ratio rather than pixels, and its rendered frame does not always follow the tier's
+    nominal size, so the frame listed for the model wins over scaling the tier's shortest side up the ratio."""
+    if not resolution or not ratio or ":" not in ratio:
+        return None
+    tier: Final = resolution.strip().lower()
+    normalized_ratio: Final = ratio.strip()
+    listed: Final = _delivered_frames(model).get(tier, MappingProxyType({})).get(normalized_ratio)
+    if listed is not None:
+        return listed
+    shortest_side: Final = _RESOLUTION_SHORTEST_SIDE.get(tier)
+    width_str, _, height_str = normalized_ratio.partition(":")
+    if shortest_side is None or not (width_str.isdigit() and height_str.isdigit()):
+        return None
+    ratio_width: Final = int(width_str)
+    ratio_height: Final = int(height_str)
+    if not ratio_width or not ratio_height:
+        return None
+    shortest_ratio_side: Final = min(ratio_width, ratio_height)
+    return f"{-(-shortest_side * ratio_width // shortest_ratio_side)}x{-(-shortest_side * ratio_height // shortest_ratio_side)}"
+
+
+def _is_input_video(item: _ContentItem) -> bool:
+    return item.type == _INPUT_VIDEO_TYPE or item.role == _INPUT_VIDEO_ROLE
+
+
+def _supplied_items(supplied: object, model: str, drop_params: bool) -> tuple[Mapping[str, object], ...]:
+    try:
+        items: Final = _CONTENT_ITEMS_ADAPTER.validate_python(supplied)
+        entries: Final = tuple(_ContentItem.model_validate(item) for item in items)
+    except ValidationError:
+        if drop_params:
+            return ()
+        raise _droppable(model, "content must be an array of Ark content items") from None
+    if any(entry.type == "text" for entry in entries) and not drop_params:
+        raise _droppable(
+            model,
+            "A text item in content is not supported, put the text in prompt since it is the only text input "
+            "guardrails scan before the request reaches Ark",
+        )
+    if any(_is_input_video(entry) for entry in entries) and not drop_params:
+        raise _droppable(model, _VIDEO_INPUT_UNSUPPORTED.format(operation="Video input (a reference_video item)"))
+    kept: Final = (item for item, entry in zip(items, entries) if entry.type != "text" and not _is_input_video(entry))
+    return tuple(MappingProxyType(item) for item in kept)
+
+
+def _reference_items(params: Mapping[str, object], model: str, drop_params: bool) -> tuple[Mapping[str, object], ...]:
+    """A caller-supplied ``content`` array wins over ``input_reference``, since its roles are the only way to express
+    reference-to-video."""
+    supplied: Final = params.get("content")
+    if supplied is not None:
+        return _supplied_items(supplied, model, drop_params)
+    reference: Final = params.get("input_reference")
+    if reference is None:
+        return ()
+    try:
+        url: Final = input_reference_url(reference)
+    except ValueError as e:
+        if drop_params:
+            return ()
+        raise _droppable(model, str(e)) from None
+    return (MappingProxyType({"type": "image_url", "image_url": {"url": url}, "role": "first_frame"}),)
+
+
+def _sent_content(
+    prompt: str, params: Mapping[str, object], model: str, drop_params: bool
+) -> tuple[Mapping[str, object], ...]:
+    """The prompt always leads, as it is the only text the input guardrails scan."""
+    return (MappingProxyType({"type": "text", "text": prompt}), *_reference_items(params, model, drop_params))
+
+
+def _seconds_label(seconds: object) -> str | None:
+    if isinstance(seconds, float) and seconds.is_integer():
+        return str(int(seconds))
+    return None if seconds is None else str(seconds)
+
+
+def _requested_usage(request_data: Mapping[str, object]) -> Mapping[str, object]:
+    """The create call is the only billed one and Ark's create response carries no usage, so the requested duration
+    and tier are what the video bills at."""
+    duration: Final = _duration_param(request_data.get("duration"))
+    resolution: Final = request_data.get("resolution")
+    return MappingProxyType(
+        {
+            key: value
+            for key, value in (
+                ("duration_seconds", float(duration) if duration is not None and duration > 0 else None),
+                ("video_resolution", resolution.strip().lower() if isinstance(resolution, str) else None),
+            )
+            if value is not None
+        }
+    )
+
+
+def _task_usage(task: _Task) -> Mapping[str, object]:
+    resolution: Final = task.resolution.strip().lower() if task.resolution else None
+    return MappingProxyType(
+        {
+            key: value
+            for key, value in (
+                ("duration_seconds", float(task.duration) if task.duration is not None else None),
+                ("video_resolution", resolution),
+                ("total_tokens", task.usage.total_tokens if task.usage is not None else None),
+            )
+            if value is not None
+        }
+    )
+
+
+def _task_status(task: _Task, raw_response: httpx.Response) -> str:
+    status: Final = _STATUS_MAP.get(task.status.strip().lower())
+    if status is None:  # defaulting it would hand back a job the caller polls forever
+        raise _video_error(raw_response, 502, f"Unknown Volcengine task status: {task.status}")
+    return status
+
+
+def _video_object_from_task(task: _Task, raw_response: httpx.Response, model: str | None) -> VideoObject:
+    status: Final = _task_status(task, raw_response)
+    usage: Final = _task_usage(task)
+    seconds: Final = usage.get("duration_seconds")
+    error: Final[dict[str, object] | None] = (  # mutable-ok: VideoObject.error is a dict field
+        {key: value for key, value in (("code", task.error.code), ("message", task.error.message)) if value is not None}
+        if task.error is not None
+        else None
+    )
+    return VideoObject(
+        id=task.id,
+        object="video",
+        status=status,
+        created_at=task.created_at,
+        completed_at=task.updated_at if status in _TERMINAL_STATUSES else None,
+        error=error,
+        seconds=_seconds_label(seconds),
+        size=_size_from_geometry(task.resolution, task.ratio, task.model),
+        model=model or task.model,
+        usage=dict(usage) if usage else None,
+    )
+
+
+def _video_url_from_task(task: _Task, raw_response: httpx.Response) -> str:
+    """A download before the video is ready is the caller polling too early, not a fault of the deployment, so it
+    answers 400, which litellm neither retries nor counts toward a deployment cooldown. A 404 would cool the
+    deployment down, and a 409 or a 5xx would be retried against a job that is simply not finished."""
+    if task.content is not None and task.content.video_url:
+        return task.content.video_url
+    status: Final = task.status.strip().lower()
+    if status in _PENDING_STATUSES:
+        raise _video_error(
+            raw_response, 400, f"Video is still processing (status: {status}). Please wait and try again."
+        )
+    if task.error is not None and (task.error.message or task.error.code):
+        raise _video_error(raw_response, 400, f"Video generation failed: {task.error.message or task.error.code}")
+    if status == "expired":
+        raise _video_error(raw_response, 400, "Task expired. Ark keeps task records for 7 days after creation.")
+    if status in ("failed", "cancelled"):
+        raise _video_error(raw_response, 400, f"Video generation ended as {status} with no video URL.")
+    raise _video_error(raw_response, 502, "Video URL not found in task response.")
+
+
+class VolcEngineVideoConfig(BaseVideoConfig):
+    """Volcengine Ark (Doubao-Seedance) on Ark's async task API: create, poll, download from the signed
+    ``content.video_url``, and list."""
+
+    def get_supported_openai_params(self, model: str) -> list[str]:  # mutable-ok: BaseVideoConfig contract returns list
+        return [
+            "model",
+            "prompt",
+            "input_reference",
+            "seconds",
+            "size",
+            "user",
+            "extra_headers",
+            "content",
+            "parameters",
+            *_CREATE_PASSTHROUGH_KEYS,
+        ]
+
+    def map_openai_params(
+        self,
+        video_create_optional_params: VideoCreateOptionalRequestParams,
+        model: str,
+        drop_params: bool,
+    ) -> dict[str, object]:  # mutable-ok: BaseVideoConfig contract
+        requested: Final = _without(video_create_optional_params, _additional_drop_keys(video_create_optional_params))
+        unsupported: Final = next(
+            (key for key, value in requested.items() if key in _UNSUPPORTED_BY_VOLCENGINE and value is not None),
+            None,
+        )
+        if unsupported is not None and not drop_params:
+            raise _droppable(model, f"'{unsupported}' is not supported by Volcengine video generation")
+        passthrough: Final = MappingProxyType(
+            {key: value for key, value in requested.items() if value is not None and key not in _MAPPED_ELSEWHERE}
+        )
+        return {
+            **_user_fields(requested.get("user"), model, drop_params),
+            **_size_fields(requested.get("size"), model, drop_params),
+            **_duration_fields(requested.get("seconds"), model, drop_params),
+            **passthrough,
+            **_parameters_fields(requested.get("parameters"), model, drop_params),
+        }
+
+    def _resolve_api_key(self, api_key: str | None) -> str:
+        resolved_api_key: Final = api_key or get_secret_str("ARK_API_KEY") or get_secret_str("VOLCENGINE_API_KEY")
+        if resolved_api_key is None:
+            raise ValueError(
+                "Volcengine API key is required. Set ARK_API_KEY or VOLCENGINE_API_KEY environment variable or pass "
+                "api_key parameter."
+            )
+        return resolved_api_key
+
+    def validate_environment(
+        self,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract; handler expects a mutable headers dict
+        model: str,
+        api_key: str | None = None,
+        litellm_params: GenericLiteLLMParams | None = None,
+    ) -> dict[str, str]:  # mutable-ok: BaseVideoConfig contract
+        resolved_api_key: Final = self._resolve_api_key(
+            api_key
+            or (litellm_params.api_key if litellm_params is not None and litellm_params.api_key else None)
+            or litellm.api_key
+        )
+        headers.update({"Authorization": f"Bearer {resolved_api_key}", "Content-Type": "application/json"})
+        return headers
+
+    def get_complete_url(
+        self,
+        model: str,
+        api_base: str | None,
+        litellm_params: dict[str, object],  # mutable-ok: BaseVideoConfig contract
+    ) -> str:
+        return _normalized_api_base(
+            get_volcengine_base_url(api_base or get_secret_str("VOLCENGINE_API_BASE") or get_secret_str("ARK_API_BASE"))
+        )
+
+    def transform_video_create_request(
+        self,
+        model: str,
+        prompt: str,
+        api_base: str,
+        video_create_optional_request_params: dict[str, object],  # mutable-ok: BaseVideoConfig contract
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[dict[str, object], RequestFiles, str]:  # mutable-ok: BaseVideoConfig contract
+        """
+        Ark bills a video when the task finishes, while litellm bills it here at create, so every request has to
+        carry a length and a tier whose cost is known now. Ark-picked length (``duration: -1``) and a video input
+        (edits, extensions and reference videos, billed at a different rate per token) are rejected for that reason,
+        and an omitted length is sent as OpenAI's documented default of 4 seconds, since the 2.5 family's own default
+        is ``-1``.
+        """
+        drop_params: Final = _drops_params(litellm_params)
+        dropped_by_config: Final = _additional_drop_keys(video_create_optional_request_params)
+        kept: Final = _without(video_create_optional_request_params, dropped_by_config)
+        params: Final = _without(
+            {
+                **{key: value for key, value in kept.items() if key != "parameters"},
+                **_parameters_fields(kept.get("parameters"), model, drop_params),
+            },
+            dropped_by_config,
+        )
+        content: Final = _sent_content(prompt, params, model, drop_params)
+        if params.get("frames") is not None and not drop_params:
+            raise _droppable(model, "frames is not supported through litellm, pass seconds instead")
+        requested_duration: Final = params.get("duration")
+        parsed_duration: Final = None if requested_duration is None else _duration_param(requested_duration)
+        if requested_duration is not None and (parsed_duration is None or parsed_duration <= 0) and not drop_params:
+            raise _droppable(
+                model,
+                f"duration must be a positive whole number of seconds, got {requested_duration!r}. Ark-picked length "
+                "(-1) is not supported through litellm, since the video is billed when the task is created, before "
+                "Ark has picked its length",
+            )
+        duration: Final = (
+            parsed_duration
+            if parsed_duration is not None and parsed_duration > 0
+            else VOLCENGINE_VIDEO_DEFAULT_DURATION_SECONDS
+        )
+
+        unknown_fields: Final = tuple(
+            sorted(key for key, value in params.items() if value is not None and key not in _CREATE_HANDLED_KEYS)
+        )
+        if unknown_fields and not drop_params:
+            raise _droppable(model, f"litellm does not send {', '.join(unknown_fields)} to Ark video generation")
+        requested_fields: Final = MappingProxyType(
+            {key: params[key] for key in _CREATE_PASSTHROUGH_KEYS if params.get(key) is not None}
+        )
+        malformed: Final = _malformed_fields(requested_fields)
+        if malformed and not drop_params:
+            raise _droppable(model, "; ".join(f"invalid {field}: {message}" for field, message in malformed))
+        resolution: Final = requested_fields.get("resolution")
+        priced_tiers: Final = _priced_tiers(_pricing_model(model, litellm_params))
+        drops_resolution: Final = (
+            priced_tiers is not None and isinstance(resolution, str) and resolution.strip().lower() not in priced_tiers
+        )
+        if drops_resolution and not drop_params:
+            raise _droppable(
+                model, f"{model} is not offered at resolution {resolution!r}, only at {', '.join(priced_tiers or ())}"
+            )
+        dropped: Final = frozenset({*(field for field, _ in malformed), *(("resolution",) if drops_resolution else ())})
+        kept_fields: Final = MappingProxyType(
+            {key: value for key, value in requested_fields.items() if key not in dropped}
+        )
+        default_tier: Final = (
+            MappingProxyType({"resolution": VOLCENGINE_VIDEO_DEFAULT_RESOLUTION})
+            if "resolution" not in kept_fields
+            and priced_tiers is not None
+            and VOLCENGINE_VIDEO_DEFAULT_RESOLUTION in priced_tiers
+            else _EMPTY_PARAMS
+        )
+
+        request_data: Final[dict[str, object]] = {  # mutable-ok: request body dict, JSON-serialized by the handler
+            "model": model,
+            "content": [dict(item) for item in content],
+            **default_tier,
+            **kept_fields,
+            "duration": duration,
+        }
+        return request_data, (), f"{api_base}{VOLCENGINE_VIDEO_TASKS_PATH}"
+
+    def transform_video_create_response(
+        self,
+        model: str,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+        custom_llm_provider: str | None = None,
+        request_data: dict[str, object] | None = None,  # mutable-ok: BaseVideoConfig contract
+    ) -> VideoObject:
+        task: Final = _parse(_TASK_CREATE_ADAPTER, raw_response)
+        if not task.id:
+            message: Final = task.error.message if task.error is not None and task.error.message else "no task id"
+            raise _video_error(raw_response, 400, f"Volcengine did not return a video task id: {message}")
+        requested: Final = request_data or _EMPTY_PARAMS
+        usage: Final = _requested_usage(requested)
+        seconds: Final = usage.get("duration_seconds")
+        requested_model, served_model = routed_video_models(logging_obj) or (model, None)
+        resolution: Final = requested.get("resolution")
+        ratio: Final = requested.get("ratio")
+        return VideoObject(
+            id=encode_video_id_with_provider(task.id, custom_llm_provider, requested_model, served_model)
+            if custom_llm_provider
+            else task.id,
+            object="video",
+            status="queued",
+            model=model,
+            seconds=_seconds_label(seconds),
+            size=_size_from_geometry(
+                resolution if isinstance(resolution, str) else None, ratio if isinstance(ratio, str) else None, model
+            ),
+            usage=dict(usage) if usage else None,
+        )
+
+    @staticmethod
+    def _task_url(video_id: str, api_base: str) -> str:
+        original_task_id: Final = extract_original_video_id(video_id)
+        encoded_task_id: Final = encode_url_path_segment(original_task_id, field_name="video_id")
+        return f"{api_base}{VOLCENGINE_VIDEO_TASKS_PATH}/{encoded_task_id}"
+
+    def transform_video_status_retrieve_request(
+        self,
+        video_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[str, dict[str, object]]:  # mutable-ok: BaseVideoConfig contract
+        return self._task_url(video_id, api_base), _EMPTY_PARAMS
+
+    def transform_video_status_retrieve_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+        custom_llm_provider: str | None = None,
+        client: "HTTPHandler | None" = None,
+    ) -> VideoObject:
+        task: Final = _parse(_TASK_ADAPTER, raw_response)
+        model, served_model = polled_video_models(logging_obj)
+        video_obj: Final = _video_object_from_task(task, raw_response, model)
+        if custom_llm_provider:
+            video_obj.id = encode_video_id_with_provider(task.id, custom_llm_provider, model, served_model)
+        return video_obj
+
+    def transform_video_content_request(
+        self,
+        video_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+        variant: str | None = None,
+    ) -> tuple[str, dict[str, object]]:  # mutable-ok: BaseVideoConfig contract
+        return self._task_url(video_id, api_base), _EMPTY_PARAMS
+
+    def transform_video_content_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+        client: "HTTPHandler | None" = None,
+    ) -> bytes:
+        video_url: Final = _video_url_from_task(_parse(_TASK_ADAPTER, raw_response), raw_response)
+        httpx_client: Final[HTTPHandler] = client if client is not None else _get_httpx_client()
+        video_response: Final = httpx_client.get(video_url)  # pyright: ignore[reportUnknownMemberType]  # HTTPHandler.get stub leaves params/headers untyped
+        video_response.raise_for_status()
+        return video_response.content
+
+    async def async_transform_video_content_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+        client: "AsyncHTTPHandler | None" = None,
+    ) -> bytes:
+        video_url: Final = _video_url_from_task(_parse(_TASK_ADAPTER, raw_response), raw_response)
+        async_httpx_client: Final[AsyncHTTPHandler] = (
+            client if client is not None else get_async_httpx_client(llm_provider=litellm.LlmProviders.VOLCENGINE)
+        )
+        video_response: Final = await async_httpx_client.get(video_url)  # pyright: ignore[reportUnknownMemberType]  # HTTPHandler.get stub leaves params/headers untyped
+        video_response.raise_for_status()
+        return video_response.content
+
+    def transform_video_remix_request(
+        self,
+        video_id: str,
+        prompt: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+        extra_body: dict[str, object] | None = None,  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[str, dict[str, object]]:  # mutable-ok: BaseVideoConfig contract
+        raise _unsupported("", _VIDEO_INPUT_UNSUPPORTED.format(operation="Video remix"))
+
+    def transform_video_remix_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+        custom_llm_provider: str | None = None,
+    ) -> VideoObject:
+        raise NotImplementedError("Video remix is not supported by Volcengine.")
+
+    def transform_video_edit_request(
+        self,
+        prompt: str,
+        video_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+        video_file: object | None = None,
+        extra_body: dict[str, object] | None = None,  # mutable-ok: BaseVideoConfig contract
+        prefetched_source_data: dict[str, object] | None = None,  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[str, Mapping[str, object], RequestFiles | None]:
+        raise _unsupported("", _VIDEO_INPUT_UNSUPPORTED.format(operation="Video edit"))
+
+    def transform_video_extension_request(
+        self,
+        prompt: str,
+        video_id: str,
+        seconds: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+        extra_body: dict[str, object] | None = None,  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[str, dict[str, object]]:  # mutable-ok: BaseVideoConfig contract
+        raise _unsupported("", _VIDEO_INPUT_UNSUPPORTED.format(operation="Video extension"))
+
+    def transform_video_list_request(
+        self,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+        after: str | None = None,
+        limit: int | None = None,
+        order: str | None = None,
+        extra_query: dict[str, object] | None = None,  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[str, dict[str, object]]:  # mutable-ok: BaseVideoConfig contract
+        """Ark pages by page_num/page_size with filters and has no cursor or order."""
+        unsupported: Final = tuple(name for name, value in (("after", after), ("order", order)) if value is not None)
+        if unsupported and not _drops_params(litellm_params):
+            raise _droppable(
+                "", f"Ark's task list pages by page_num in extra_query and has no {' or '.join(unsupported)}"
+            )
+        return f"{api_base}{VOLCENGINE_VIDEO_TASKS_PATH}", {
+            **({"page_size": str(limit)} if limit is not None else {}),
+            **(extra_query or {}),
+        }
+
+    def transform_video_list_response(  # pyright: ignore[reportIncompatibleMethodOverride]  # base declares dict[str, str] but the payload is a heterogeneous list body
+        self,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+        custom_llm_provider: str | None = None,
+    ) -> dict[str, object]:  # mutable-ok: OpenAI list body served as JSON by the proxy
+        list_payload: Final = _parse(_TASK_LIST_ADAPTER, raw_response)
+        videos: Final = tuple(_video_object_from_task(task, raw_response, None) for task in list_payload.items)
+        data: Final = [
+            video.model_dump()
+            | (
+                {"id": encode_video_id_with_provider(video.id, custom_llm_provider, video.model)}
+                if custom_llm_provider
+                else {}
+            )
+            for video in videos
+        ]
+        return {
+            "object": "list",
+            "data": data,
+            "total": list_payload.total if list_payload.total is not None else len(data),
+            "has_more": False,
+            **({"first_id": data[0]["id"], "last_id": data[-1]["id"]} if data else {}),
+        }
+
+    def transform_video_delete_request(
+        self,
+        video_id: str,
+        api_base: str,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict[str, str],  # mutable-ok: BaseVideoConfig contract
+    ) -> tuple[str, dict[str, object]]:  # mutable-ok: BaseVideoConfig contract
+        raise NotImplementedError("Video delete is not supported by Volcengine.")
+
+    def transform_video_delete_response(
+        self,
+        raw_response: httpx.Response,
+        logging_obj: "Logging",
+    ) -> VideoObject:
+        raise NotImplementedError("Video delete is not supported by Volcengine.")
+
+    def get_error_class(
+        self,
+        error_message: str,
+        status_code: int,
+        headers: dict[str, str] | httpx.Headers,  # mutable-ok: BaseVideoConfig contract
+    ) -> BaseLLMException:
+        return VolcEngineVideoError(status_code=status_code, message=error_message, headers=headers)

litellm/llms/watsonx/common_utils.py

@@ -141,7 +141,7 @@ async def _aconvert_watsonx_messages_core(
         model_prompt_dict: Final = custom_prompt_dict[model]
         return ptf.custom_prompt(
             messages=messages,
-            role_dict=model_prompt_dict.get("role_dict", model_prompt_dict.get("roles")),
+            role_dict=model_prompt_dict.get("role_dict", model_prompt_dict.get("roles", {})),
             initial_prompt_value=model_prompt_dict.get("initial_prompt_value", ""),
             final_prompt_value=model_prompt_dict.get("final_prompt_value", ""),
             bos_token=model_prompt_dict.get("bos_token", ""),
@@ -173,7 +173,7 @@ def _convert_watsonx_messages_core(
         model_prompt_dict: Final = custom_prompt_dict[model]
         return ptf.custom_prompt(
             messages=messages,
-            role_dict=model_prompt_dict.get("role_dict", model_prompt_dict.get("roles")),
+            role_dict=model_prompt_dict.get("role_dict", model_prompt_dict.get("roles", {})),
             initial_prompt_value=model_prompt_dict.get("initial_prompt_value", ""),
             final_prompt_value=model_prompt_dict.get("final_prompt_value", ""),
             bos_token=model_prompt_dict.get("bos_token", ""),

litellm/llms/xai/realtime/transformation.py

@@ -16,6 +16,7 @@ construction time (see ``handler.py``) so all normalization is isolated here
 and ``RealTimeStreaming`` stays provider-agnostic.
 """
 
+from collections.abc import Mapping
 from typing import Final
 
 
@@ -77,6 +78,18 @@ class XAIRealtimeNormalizer:
         event = self._normalize_response_usage_event(event, event_type)
         return event
 
+    def gate_turn_detection(self, _turn_detection: object, /) -> Mapping[str, object]:
+        """xAI auto-answers under ``server_vad`` even with ``create_response`` off, so the gate is always untyped."""
+        return {"create_response": False}
+
+    @property
+    def deletes_conversation_items(self) -> bool:
+        """xAI answers ``conversation.item.delete`` with ``Item not found`` until it commits the audio item.
+
+        The guardrail blocks on an interim transcript, before that commit.
+        """
+        return False
+
     def patch_outgoing_session(self, session: dict) -> dict:
         """Patch a client ``session.update`` payload before forwarding to xAI.
 

litellm/main.py

@@ -52,6 +52,7 @@ from litellm.utils import (
     get_litellm_params,
     get_optional_params,
     peek_reasoning_summary_aliases,
+    reasoning_effort_with_summary_alias,
     strip_reasoning_summary_aliases_from_optional_params,
 )
 
@@ -5681,7 +5682,15 @@ def completion(
             "allowed_openai_params": allowed_openai_params,
             "base_model": base_model,
         }
-        optional_params = get_optional_params(**optional_param_args, **non_default_params)
+        provider_reasoning_effort: Final = reasoning_effort_with_summary_alias(
+            reasoning_effort,
+            non_default_params,
+            accepts_summary=provider_config is not None
+            and provider_config.accepts_reasoning_summary_in_reasoning_effort,
+        )
+        optional_params = get_optional_params(
+            **{**optional_param_args, "reasoning_effort": provider_reasoning_effort}, **non_default_params
+        )
         processed_non_default_params: Final = pre_process_non_default_params(
             model=model,
             passed_params=optional_param_args,

litellm/model_prices_and_context_window_backup.json

@@ -2773,7 +2773,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2808,7 +2808,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2843,7 +2843,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2878,7 +2878,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2913,7 +2913,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2948,7 +2948,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -3743,7 +3743,7 @@
         "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -15494,7 +15494,7 @@
         },
         "supports_adaptive_thinking": true,
         "supports_legacy_thinking": true,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -20832,7 +20832,7 @@
         "output_cost_per_token": 2.5000010000000002e-05,
         "output_dbu_cost_per_token": 0.000357143,
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_legacy_thinking": true,
         "supports_prompt_caching": true,
@@ -21047,7 +21047,7 @@
         "output_cost_per_token": 1.5000020000000002e-05,
         "output_dbu_cost_per_token": 0.000214286,
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_legacy_thinking": true,
         "supports_prompt_caching": true,
@@ -49176,7 +49176,7 @@
         "max_tokens": 64000,
         "mode": "chat",
         "output_cost_per_token": 2.5e-05,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_prompt_caching": true,
@@ -51006,7 +51006,7 @@
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "output_cost_per_token_batches": 7.5e-06,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -51179,7 +51179,7 @@
         "supported_regions": [
             "global"
         ],
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
@@ -51200,7 +51200,7 @@
         "supported_regions": [
             "us-central1"
         ],
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
@@ -58822,7 +58822,7 @@
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "output_cost_per_token_batches": 7.5e-06,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -60467,6 +60467,61 @@
             }
         ]
     },
+    "volcengine/doubao-seedance-2-5-260628": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.216,
+        "output_cost_per_second_480p": 0.096075,
+        "output_cost_per_second_720p": 0.216,
+        "output_cost_per_second_1080p": 0.5346,
+        "supported_modalities": ["text", "image", "video", "audio"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "854x480", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "480x854", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s). List 70 CNY per million tokens at 480p and 720p and 77 at 1080p without a video input, matching Ark's per-resolution table and its 1080p 16:9 example of 3.74 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Accepts up to 50 reference assets (30 images, 10 videos, 10 audios) and clips of 4-30s; per-asset billing beyond the free allowance is not encoded. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
+    "volcengine/doubao-seedance-2-0-260128": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.141942857,
+        "output_cost_per_second_480p": 0.066003429,
+        "output_cost_per_second_720p": 0.141942857,
+        "output_cost_per_second_1080p": 0.354085714,
+        "output_cost_per_second_4k": 0.722057143,
+        "supported_modalities": ["text", "image", "video", "audio"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 46 CNY per million tokens at 480p and 720p, 51 at 1080p, and 26 at 4k without a video input, matching Ark's per-resolution table and its 1080p and 4k 16:9 examples of 2.48 and 5.05 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s; 4k is exclusive to this tier. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
+    "volcengine/doubao-seedance-2-0-fast-260128": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.114171429,
+        "output_cost_per_second_480p": 0.053089714,
+        "output_cost_per_second_720p": 0.114171429,
+        "supported_modalities": ["text", "image", "video", "audio"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 37 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
+    "volcengine/doubao-seedance-2-0-mini-260615": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.070971429,
+        "output_cost_per_second_480p": 0.033001714,
+        "output_cost_per_second_720p": 0.070971429,
+        "supported_modalities": ["text", "image"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 23 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Only text and image inputs are declared; the mini tier's reference video and audio caps were not verified. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
     "bedrock/us-east-1/zai.glm-5": {
         "input_cost_per_token": 1e-06,
         "output_cost_per_token": 3.2e-06,
@@ -72334,7 +72389,7 @@
         "max_output_tokens": 64000,
         "max_tokens": 64000,
         "mode": "chat",
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,

litellm/proxy/anthropic_endpoints/endpoints.py

@@ -29,6 +29,7 @@ from litellm.proxy.common_request_processing import (
     create_response,
     log_llm_api_exception,
     proxy_exception_from_http_exception,
+    proxy_exception_from_route_error,
     resolve_litellm_call_id,
 )
 from litellm.proxy.common_utils.error_body_call_id import error_body_call_id
@@ -339,10 +340,10 @@ async def count_tokens(
         messages: Final = data.get("messages", [])
 
         if not model_name:
-            raise HTTPException(status_code=400, detail={"error": "model parameter is required"})
+            raise HTTPException(status_code=400, detail="model parameter is required")
 
         if not messages:
-            raise HTTPException(status_code=400, detail={"error": "messages parameter is required"})
+            raise HTTPException(status_code=400, detail="messages parameter is required")
 
         # Create TokenCountRequest for the internal endpoint
         from litellm.proxy._types import TokenCountRequest
@@ -360,13 +361,13 @@ async def count_tokens(
             messages_format="anthropic",
             output_format=_requested_output_format(data),
         )
-    except HTTPException:
-        raise
+    except HTTPException as e:
+        return _anthropic_error_json_response(proxy_exception_from_http_exception(e, {}), request)
     except ProxyException as e:
         return _anthropic_error_json_response(e, request)
     except Exception as e:
         log_llm_api_exception(e, litellm_call_id)
-        raise HTTPException(status_code=500, detail={"error": f"Internal server error: {e}"})
+        return _anthropic_error_json_response(proxy_exception_from_route_error(e), request)
     else:
         return {"input_tokens": token_response.total_tokens}
 

litellm/proxy/google_endpoints/endpoints.py

@@ -1,16 +1,22 @@
 from collections.abc import Mapping
 from itertools import chain
+from types import MappingProxyType
 from typing import Final
 
 from fastapi import APIRouter, Depends, HTTPException, Request, Response
-from fastapi.responses import ORJSONResponse
+from fastapi.responses import JSONResponse, ORJSONResponse
 from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter, ValidationError
 
 from litellm.litellm_core_utils.json_validation_rule import normalize_json_schema_types
 from litellm.proxy._types import *
-from litellm.proxy._types import TokenCountRequest
+from litellm.proxy._types import ProxyException, TokenCountRequest
 from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth
-from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing
+from litellm.proxy.common_request_processing import (
+    ProxyBaseLLMRequestProcessing,
+    log_llm_api_exception,
+    proxy_exception_from_http_exception,
+    resolve_litellm_call_id,
+)
 from litellm.proxy.common_utils.http_parsing_utils import (  # noqa: F401  # legacy module exports
     _read_request_body,  # pyright: ignore[reportPrivateUsage,reportUnusedImport]  # backwards-compatible package export
     read_request_body,
@@ -63,6 +69,37 @@ def _input_schema(declaration: _GoogleCountTokensFunctionDeclaration) -> object:
     return {"type": "object", "properties": {}}
 
 
+_GOOGLE_RPC_STATUS: Final = MappingProxyType(
+    {
+        400: "INVALID_ARGUMENT",
+        401: "UNAUTHENTICATED",
+        403: "PERMISSION_DENIED",
+        404: "NOT_FOUND",
+        409: "ABORTED",
+        429: "RESOURCE_EXHAUSTED",
+        499: "CANCELLED",
+        501: "UNIMPLEMENTED",
+        503: "UNAVAILABLE",
+        504: "DEADLINE_EXCEEDED",
+    }
+)
+
+
+def _google_error_response(request: Request, error: ProxyException) -> JSONResponse:
+    from litellm.proxy.proxy_server import (
+        _close_dangling_otel_server_span,  # pyright: ignore[reportPrivateUsage]  # proxy_server keeps the span-close helper private; error JSONResponses returned by the route must stamp the OTel server span like the global ProxyException handler does
+    )
+
+    status_code: Final = int(error.code) if error.code is not None and error.code.isdigit() else 500
+    _close_dangling_otel_server_span(request, status_code, exc=error)
+    status: Final = _GOOGLE_RPC_STATUS.get(status_code, "INTERNAL" if status_code >= 500 else "FAILED_PRECONDITION")
+    return JSONResponse(
+        status_code=status_code,
+        content={"error": {"code": status_code, "message": error.message, "status": status}},
+        headers=error.headers,
+    )
+
+
 def _google_count_tokens_request(model_name: str, data: object) -> TokenCountRequest:
     from litellm.google_genai.adapters.transformation import GoogleGenAIAdapter
 
@@ -251,16 +288,26 @@ async def google_count_tokens(request: Request, model_name: str):
     data: Final = await read_request_body(request=request)
     try:
         token_request: Final = _google_count_tokens_request(model_name, data)
-    except ValidationError as error:
-        raise HTTPException(status_code=400, detail="Invalid Google countTokens request body") from error
+    except ValidationError:
+        return _google_error_response(
+            request,
+            ProxyException(
+                message="Invalid Google countTokens request body", type="invalid_request_error", param=None, code=400
+            ),
+        )
 
-    # Call the internal token counter function with direct request flag set to False
-    token_response: Final = await internal_token_counter(
-        request=token_request,
-        call_endpoint=True,
-    )
-    if token_response is None:
-        return TokenCountDetailsResponse(totalTokens=0, promptTokensDetails=[])
+    try:
+        token_response: Final = await internal_token_counter(request=token_request, call_endpoint=True)
+    except ProxyException as error:
+        return _google_error_response(request, error)
+    except HTTPException as error:
+        return _google_error_response(request, proxy_exception_from_http_exception(error, {}))
+    except Exception as error:
+        log_llm_api_exception(error, resolve_litellm_call_id(request.headers.get("x-litellm-call-id")))
+        return _google_error_response(
+            request,
+            ProxyException(message="Internal server error", type="internal_server_error", param=None, code=500),
+        )
     original_response: Final[dict] = token_response.original_response or {}
     return TokenCountDetailsResponse(
         totalTokens=original_response.get("totalTokens") or token_response.total_tokens or 0,

litellm/proxy/guardrails/guardrail_endpoints.py

@@ -1483,6 +1483,17 @@ async def get_guardrail_info(guardrail_id: str):
         raise HTTPException(status_code=500, detail=str(e))
 
 
+def _undeclared_logging_only(
+    guardrail_class: type[CustomGuardrail], hooks: Sequence[GuardrailEventHooks]
+) -> tuple[GuardrailEventHooks, ...]:
+    """logging_only when the class accepts it at save time without declaring it in its hooks."""
+    return (
+        (GuardrailEventHooks.logging_only,)
+        if guardrail_class.accepts_logging_only_mode() and GuardrailEventHooks.logging_only not in hooks
+        else ()
+    )
+
+
 @router.get(
     "/guardrails/ui/add_guardrail_settings",
     tags=["Guardrails"],
@@ -1515,7 +1526,7 @@ async def get_guardrail_ui_settings() -> GuardrailUIAddGuardrailSettings:
     ]
 
     supported_modes_by_provider: Final = {
-        provider: [hook.value for hook in hooks]
+        provider: [hook.value for hook in (*hooks, *_undeclared_logging_only(guardrail_class, hooks))]
         for provider, guardrail_class in guardrail_class_registry.items()
         if (hooks := guardrail_class.get_supported_event_hooks()) is not None
     } | MappingProxyType(

litellm/proxy/guardrails/guardrail_hooks/litellm_content_filter/content_filter.py

@@ -6,6 +6,7 @@ to detect and block/mask sensitive content.
 """
 
 import asyncio
+import contextvars
 import itertools
 import json
 import os
@@ -50,6 +51,7 @@ from litellm.types.utils import (
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
 
 from litellm.types.guardrails import (
     BlockedWord,
@@ -101,6 +103,13 @@ WORD_NUMBER_SEQUENCE_PATTERN: Final = re.compile(
 )
 WORD_NUMBER_TOKEN_FINDER: Final = re.compile(rf"(?:{WORD_NUMBER_TOKEN_REGEX})", re.IGNORECASE)
 
+_logging_only_scan_guardrail: Final[contextvars.ContextVar["ContentFilterGuardrail | None"]] = contextvars.ContextVar(
+    "litellm_content_filter_logging_only_scan", default=None
+)
+_logging_only_scan_would_reject: Final[contextvars.ContextVar[bool]] = contextvars.ContextVar(
+    "litellm_content_filter_logging_only_scan_would_reject", default=False
+)
+
 
 class ConditionalCategoryConfig(TypedDict):
     identifier_words: Sequence[str]
@@ -1209,6 +1218,18 @@ class ContentFilterGuardrail(CustomGuardrail):
                 return (keyword, action, description)
         return None
 
+    def _is_logging_only_scan(self) -> bool:
+        return _logging_only_scan_guardrail.get() is self
+
+    def _enforces_block(self, action: ContentFilterAction) -> bool:
+        return action == ContentFilterAction.BLOCK and not self._is_logging_only_scan()
+
+    @staticmethod
+    def _asks_to_block(detection: ContentFilterDetection) -> bool:
+        if detection["type"] == "competitor_intent":
+            return detection["action_hint"] in ("refuse", "reframe")
+        return detection["action"] == ContentFilterAction.BLOCK.value
+
     def _handle_conditional_match(
         self,
         matched_phrase: str,
@@ -1228,7 +1249,7 @@ class ContentFilterGuardrail(CustomGuardrail):
             }
             detections.append(category_detection)
 
-        if action == ContentFilterAction.BLOCK:
+        if self._enforces_block(action):
             error_msg: Final = (
                 f"Content blocked: {category_name} conditional match '{matched_phrase}' detected (severity: {severity})"
             )
@@ -1269,7 +1290,7 @@ class ContentFilterGuardrail(CustomGuardrail):
             }
             detections.append(category_detection)
 
-        if action == ContentFilterAction.BLOCK:
+        if self._enforces_block(action):
             error_msg = f"Content blocked: {category_name} category keyword '{keyword}' detected (severity: {severity})"
             verbose_proxy_logger.warning(error_msg)
             raise HTTPException(
@@ -1312,7 +1333,7 @@ class ContentFilterGuardrail(CustomGuardrail):
             }
             detections.append(pattern_detection)
 
-        if action == ContentFilterAction.BLOCK:
+        if self._enforces_block(action):
             error_msg: Final = f"Content blocked: {pattern_name} pattern detected"
             verbose_proxy_logger.warning(error_msg)
             raise HTTPException(
@@ -1346,7 +1367,7 @@ class ContentFilterGuardrail(CustomGuardrail):
             }
             detections.append(blocked_word_detection)
 
-        if action == ContentFilterAction.BLOCK:
+        if self._enforces_block(action):
             error_msg = f"Content blocked: keyword '{keyword}' detected"
             if description:
                 error_msg += f" ({description})"
@@ -1619,7 +1640,7 @@ class ContentFilterGuardrail(CustomGuardrail):
         - Number of entities masked
         - Whether the guardrail blocked the request (max risk)
         """
-        if status == "guardrail_intervened":
+        if status in ("guardrail_intervened", "guardrail_flagged"):
             return 10.0
 
         total_masked: Final = sum(masked_entity_count.values()) if masked_entity_count else 0
@@ -1665,6 +1686,8 @@ class ContentFilterGuardrail(CustomGuardrail):
         }
         detections.append(detection)
 
+        if self._is_logging_only_scan():
+            return
         if action_hint_val == "refuse":
             msg = "Content blocked: competitor comparison or ranking intent detected."
             if self._competitor_intent_checker and getattr(
@@ -1763,18 +1786,18 @@ class ContentFilterGuardrail(CustomGuardrail):
             return raw_name
         return None
 
+    def _reject(self, error: str) -> None:
+        """Raise the 400 an enforcing mode returns, or note it on the leg a logging_only scan is observing."""
+        if self._is_logging_only_scan():
+            _logging_only_scan_would_reject.set(True)
+            return
+        raise HTTPException(status_code=400, detail={"error": error})
+
     def _assert_argument_label_clean(
         self, text: str, detections: list[ContentFilterDetection], context_label: str
     ) -> None:
         if self._filter_single_text(text, detections=detections) != text:
-            raise HTTPException(
-                status_code=400,
-                detail={
-                    "error": (
-                        f"Content blocked: {context_label} argument matched a masking rule on a non-rewritable field"
-                    )
-                },
-            )
+            self._reject(f"Content blocked: {context_label} argument matched a masking rule on a non-rewritable field")
 
     def _filter_argument_value(
         self,
@@ -1784,10 +1807,8 @@ class ContentFilterGuardrail(CustomGuardrail):
         depth: int = 0,
     ) -> object:
         if depth > DEFAULT_MAX_RECURSE_DEPTH:
-            raise HTTPException(
-                status_code=400,
-                detail={"error": f"Content blocked: {context_label} arguments exceed the maximum nesting depth"},
-            )
+            self._reject(f"Content blocked: {context_label} arguments exceed the maximum nesting depth")
+            return value
         if isinstance(value, str):
             return self._filter_single_text(value, detections=detections)
         if isinstance(value, (int, float)) and not isinstance(value, bool):
@@ -1874,6 +1895,21 @@ class ContentFilterGuardrail(CustomGuardrail):
             if filtered_arguments != arguments:
                 self._set_tool_call_arguments(tool_call, filtered_arguments)
 
+    async def _scan_logged_call(
+        self,
+        kwargs: dict[str, object],  # mutable-ok: CustomLogger.async_logging_hook contract
+        response: object | None,
+        translation: "BaseTranslation",
+        output_translation: "BaseTranslation",
+        scratch_metadata: dict[str, object],  # mutable-ok: apply_guardrail records its verdict into request metadata
+    ) -> None:
+        """Scan the logged call with every BLOCK match recorded instead of raised."""
+        token: Final = _logging_only_scan_guardrail.set(self)
+        try:
+            await super()._scan_logged_call(kwargs, response, translation, output_translation, scratch_metadata)
+        finally:
+            _logging_only_scan_guardrail.reset(token)
+
     async def apply_guardrail(
         self,
         inputs: "GenericGuardrailAPIInputs",
@@ -1904,6 +1940,7 @@ class ContentFilterGuardrail(CustomGuardrail):
         masked_entity_count: Final[dict[str, int]] = {}
         status: GuardrailStatus = "success"
         exception_str: str = ""
+        would_reject_token: Final = _logging_only_scan_would_reject.set(False)
 
         try:
             texts: Final = inputs.get("texts", [])
@@ -1947,11 +1984,13 @@ class ContentFilterGuardrail(CustomGuardrail):
             exception_str = str(e)
             raise e
         finally:
+            would_reject: Final = _logging_only_scan_would_reject.get() or any(map(self._asks_to_block, detections))
+            _logging_only_scan_would_reject.reset(would_reject_token)
             # Log guardrail information
             self._log_guardrail_information(
                 request_data=request_data,
                 detections=detections,
-                status=status,
+                status="guardrail_flagged" if status == "success" and would_reject else status,
                 start_time=start_time,
                 masked_entity_count=masked_entity_count,
                 exception_str=exception_str,

litellm/proxy/hooks/key_management_event_hooks.py

@@ -57,13 +57,6 @@ class KeyManagementEventHooks:
         )
         from litellm.proxy.proxy_server import litellm_proxy_admin_name
 
-        # Send email notification - non-blocking, independent operation
-        if data.send_invite_email is True:
-            try:
-                await KeyManagementEventHooks._send_key_created_email(response.model_dump(exclude_none=True))
-            except Exception as e:
-                verbose_proxy_logger.warning("Failed to send key created email: %s", e)
-
         if is_audit_logging_enabled():
             _updated_values: Final = response.model_dump_json(exclude_none=True)
             asyncio.create_task(
@@ -86,6 +79,13 @@ class KeyManagementEventHooks:
                 )
             )
 
+        # Send email notification - non-blocking, independent operation
+        if data.send_invite_email is True:
+            try:
+                await KeyManagementEventHooks._send_key_created_email(response.model_dump(exclude_none=True))
+            except Exception as e:
+                verbose_proxy_logger.warning("Failed to send key created email: %s", e)
+
         # Store the generated key in the secret manager - non-blocking, independent operation
         try:
             await KeyManagementEventHooks._store_virtual_key_in_secret_manager(
@@ -118,16 +118,6 @@ class KeyManagementEventHooks:
         )
         from litellm.proxy.proxy_server import litellm_proxy_admin_name
 
-        if data.key_alias is not None and data.key_alias != existing_key_row.key_alias:
-            try:
-                await KeyManagementEventHooks._rename_virtual_key_in_secret_manager(
-                    current_secret_name=existing_key_row.key_alias or f"virtual-key-{existing_key_row.token}",
-                    new_secret_name=data.key_alias,
-                    team_id=existing_key_row.team_id,
-                )
-            except Exception as e:
-                verbose_proxy_logger.warning("Failed to rename virtual key in secret manager: %s", e)
-
         if is_audit_logging_enabled():
             updated_fields: Final = {
                 **data.model_dump(exclude_none=True),
@@ -156,6 +146,16 @@ class KeyManagementEventHooks:
             )
             asyncio.create_task(create_audit_log_for_update(request_data=request_data))
 
+        if data.key_alias is not None and data.key_alias != existing_key_row.key_alias:
+            try:
+                await KeyManagementEventHooks._rename_virtual_key_in_secret_manager(
+                    current_secret_name=existing_key_row.key_alias or f"virtual-key-{existing_key_row.token}",
+                    new_secret_name=data.key_alias,
+                    team_id=existing_key_row.team_id,
+                )
+            except Exception as e:
+                verbose_proxy_logger.warning("Failed to rename virtual key in secret manager: %s", e)
+
     @staticmethod
     async def async_key_rotated_hook(
         data: RegenerateKeyRequest | None,
@@ -171,6 +171,28 @@ class KeyManagementEventHooks:
         )
         from litellm.proxy.proxy_server import litellm_proxy_admin_name
 
+        # store the audit log
+        if is_audit_logging_enabled() and existing_key_row.token is not None:
+            asyncio.create_task(
+                create_audit_log_for_update(
+                    request_data=LiteLLM_AuditLogs(
+                        id=str(uuid.uuid4()),
+                        updated_at=datetime.now(timezone.utc),
+                        changed_by=get_audit_log_changed_by(
+                            litellm_changed_by=litellm_changed_by,
+                            user_api_key_dict=user_api_key_dict,
+                            litellm_proxy_admin_name=litellm_proxy_admin_name,
+                        ),
+                        changed_by_api_key=user_api_key_dict.token,
+                        table_name=LitellmTableNames.KEY_TABLE_NAME,
+                        object_id=existing_key_row.token,
+                        action="rotated",
+                        updated_values=response.model_dump_json(exclude_none=True),
+                        before_value=existing_key_row.model_dump_json(exclude_none=True),
+                    )
+                )
+            )
+
         # Store the generated key in the secret manager - non-blocking, independent operation
         if response.token_id is not None:
             try:
@@ -204,28 +226,6 @@ class KeyManagementEventHooks:
         except Exception as e:
             verbose_proxy_logger.warning("Failed to send key rotated email: %s", e)
 
-        # store the audit log
-        if is_audit_logging_enabled() and existing_key_row.token is not None:
-            asyncio.create_task(
-                create_audit_log_for_update(
-                    request_data=LiteLLM_AuditLogs(
-                        id=str(uuid.uuid4()),
-                        updated_at=datetime.now(timezone.utc),
-                        changed_by=get_audit_log_changed_by(
-                            litellm_changed_by=litellm_changed_by,
-                            user_api_key_dict=user_api_key_dict,
-                            litellm_proxy_admin_name=litellm_proxy_admin_name,
-                        ),
-                        changed_by_api_key=user_api_key_dict.token,
-                        table_name=LitellmTableNames.KEY_TABLE_NAME,
-                        object_id=existing_key_row.token,
-                        action="rotated",
-                        updated_values=response.model_dump_json(exclude_none=True),
-                        before_value=existing_key_row.model_dump_json(exclude_none=True),
-                    )
-                )
-            )
-
     @staticmethod
     async def async_key_deleted_hook(
         data: KeyRequest,

litellm/proxy/hooks/user_management_event_hooks.py

@@ -23,6 +23,7 @@ from litellm.proxy._types import (
 from litellm.proxy.management_helpers.audit_logs import (
     create_audit_log_for_update,
     is_audit_logging_enabled,
+    track_audit_task,
 )
 from litellm.repositories.user_repository import UserRepository
 
@@ -41,7 +42,18 @@ class UserManagementEventHooks:
         - Creating an audit log for the user creation
         - Sending a user invitation email to the user
         """
-        from litellm.proxy.proxy_server import litellm_proxy_admin_name, prisma_client
+        from litellm.proxy.proxy_server import litellm_proxy_admin_name
+
+        if is_audit_logging_enabled():
+            track_audit_task(
+                asyncio.create_task(
+                    UserManagementEventHooks._create_user_created_audit_log(
+                        response=response,
+                        user_api_key_dict=user_api_key_dict,
+                        litellm_proxy_admin_name=litellm_proxy_admin_name,
+                    )
+                )
+            )
 
         #########################################################
         ########## Send User Invitation Email ################
@@ -52,9 +64,14 @@ class UserManagementEventHooks:
             user_api_key_dict=user_api_key_dict,
         )
 
-        #########################################################
-        ########## CREATE AUDIT LOG ################
-        #########################################################
+    @staticmethod
+    async def _create_user_created_audit_log(
+        response: NewUserResponse,
+        user_api_key_dict: UserAPIKeyAuth,
+        litellm_proxy_admin_name: str | None,
+    ) -> None:
+        from litellm.proxy.proxy_server import prisma_client
+
         try:
             if prisma_client is None:
                 raise Exception(CommonProxyErrors.db_not_connected_error.value)
@@ -63,16 +80,14 @@ class UserManagementEventHooks:
             user_row: Final = await UserRepository(prisma_client).find_by_id(response.user_id)
             if user_row is None:
                 raise Exception(f"no user row found for user_id={response.user_id}")
-            asyncio.create_task(
-                UserManagementEventHooks.create_internal_user_audit_log(
-                    user_id=user_row.user_id,
-                    action="created",
-                    litellm_changed_by=user_api_key_dict.user_id,
-                    user_api_key_dict=user_api_key_dict,
-                    litellm_proxy_admin_name=litellm_proxy_admin_name,
-                    before_value=None,
-                    after_value=user_row.model_dump_json(exclude_none=True),
-                )
+            await UserManagementEventHooks.create_internal_user_audit_log(
+                user_id=user_row.user_id,
+                action="created",
+                litellm_changed_by=user_api_key_dict.user_id,
+                user_api_key_dict=user_api_key_dict,
+                litellm_proxy_admin_name=litellm_proxy_admin_name,
+                before_value=None,
+                after_value=user_row.model_dump_json(exclude_none=True),
             )
         except Exception as e:
             verbose_proxy_logger.warning("Unable to create audit log for user on `/user/new` - %s", e)

litellm/proxy/litellm_pre_call_utils.py

@@ -27,6 +27,7 @@ from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm._service_logger import ServiceLogging
 from litellm._uuid import uuid
 from litellm.constants import (
+    CALLER_SESSION_ID_METADATA_KEY,
     CLIENT_OUTPUT_CEILING_METADATA_KEY,
     CONSUMED_REQUEST_TAGS_METADATA_KEY,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
@@ -187,6 +188,30 @@ def _caller_trace_field(data: Mapping[str, object], metadata_variable_name: str,
     return requester_value if isinstance(requester_value, str) and requester_value else None
 
 
+def _caller_body_session_id(data: Mapping[str, object], metadata_variable_name: str) -> str | None:
+    top_level: Final = data.get("litellm_session_id")
+    if isinstance(top_level, str) and top_level:
+        return top_level
+    return _caller_trace_field(data, metadata_variable_name, "session_id")
+
+
+def _caller_session_beside_trace_header(
+    normalized_headers: Mapping[str, str], data: Mapping[str, object], metadata_variable_name: str
+) -> str | None:
+    session_headers: Final = {key: value for key, value in normalized_headers.items() if key != "x-litellm-trace-id"}
+    return get_chain_id_from_headers(session_headers) or _caller_body_session_id(data, metadata_variable_name)
+
+
+def _drop_client_caller_session(request_body: Mapping[str, object]) -> None:
+    for metadata_key in ("metadata", "litellm_metadata"):
+        if isinstance(metadata := request_body.get(metadata_key), dict):
+            metadata.pop(CALLER_SESSION_ID_METADATA_KEY, None)
+
+
+def _requester_metadata_snapshot(metadata: Mapping[str, object]) -> Mapping[str, object]:
+    return copy.deepcopy({key: value for key, value in metadata.items() if key != CALLER_SESSION_ID_METADATA_KEY})
+
+
 def _stampable_key_hash(user_api_key_dict: UserAPIKeyAuth) -> str | None:
     """Only proxy-validated keys are stamped, proven by the unforgeable
     via_virtual_key marker AND a known non-secret shape: the sha256 hex digest
@@ -1623,6 +1648,8 @@ class LiteLLMProxyRequestSetup:
         from litellm.proxy._types import LitellmMetadataFromRequestHeaders
 
         metadata_from_headers: Final = LitellmMetadataFromRequestHeaders()
+        request_body: Final = cast(Mapping[str, object], data)  # cast-ok: request body is a str-keyed JSON object
+        _drop_client_caller_session(request_body)
         spend_logs_metadata: Final = LiteLLMProxyRequestSetup._get_spend_logs_metadata_from_request_headers(headers)
         if spend_logs_metadata is not None:
             metadata_from_headers["spend_logs_metadata"] = spend_logs_metadata
@@ -1631,6 +1658,9 @@ class LiteLLMProxyRequestSetup:
         # Finally update the requests metadata with the `metadata_from_headers`
         #########################################################################################
 
+        normalized_headers: Final = MappingProxyType(
+            {k.lower(): v for k, v in headers.items() if isinstance(k, str) and isinstance(v, str)}
+        )
         agent_id_from_header: Final = headers.get("x-litellm-agent-id")
         # Explicit litellm headers take precedence; fall back to any x-*-session-id header.
         chain_id: Final = get_chain_id_from_headers(dict(headers))
@@ -1639,6 +1669,13 @@ class LiteLLMProxyRequestSetup:
             metadata_from_headers["agent_id"] = agent_id_from_header
             verbose_proxy_logger.debug("Extracted agent_id from header: %s", agent_id_from_header)
 
+        spend_log_session_id: Final = (
+            _caller_session_beside_trace_header(normalized_headers, request_body, _metadata_variable_name)
+            if litellm.request_correlation_in_logs
+            and chain_id is not None
+            and chain_id == normalized_headers.get("x-litellm-trace-id")
+            else None
+        )
         if chain_id:
             metadata_from_headers["trace_id"] = chain_id
             metadata_from_headers["session_id"] = chain_id
@@ -1663,7 +1700,6 @@ class LiteLLMProxyRequestSetup:
         # - but lets a caller's existing traceparent/baggage headers (from
         # real OTel instrumentation) correlate with litellm's own logs instead
         # of generating an unrelated trace_id.
-        normalized_headers: Final = MappingProxyType({k.lower(): v for k, v in headers.items() if isinstance(k, str)})
         if "litellm_trace_id" not in data:
             traceparent: Final = normalized_headers.get("traceparent")
             if isinstance(traceparent, str):
@@ -1689,11 +1725,7 @@ class LiteLLMProxyRequestSetup:
             baggage: Final = normalized_headers.get("baggage")
             if isinstance(baggage, str):
                 session_id_from_baggage: Final = _session_id_from_baggage(baggage)
-                caller_session_id: Final = _caller_trace_field(
-                    cast(Mapping[str, object], data),  # cast-ok: request body is a str-keyed JSON object
-                    _metadata_variable_name,
-                    "session_id",
-                )
+                caller_session_id: Final = _caller_trace_field(request_body, _metadata_variable_name, "session_id")
                 if session_id_from_baggage and (
                     caller_session_id is None or caller_session_id == session_id_from_baggage
                 ):
@@ -1702,7 +1734,11 @@ class LiteLLMProxyRequestSetup:
                     verbose_proxy_logger.debug("Extracted session_id from W3C baggage header")
 
         if isinstance(data[_metadata_variable_name], dict):
-            data[_metadata_variable_name].update(metadata_from_headers)
+            data[_metadata_variable_name].update(
+                {**metadata_from_headers, CALLER_SESSION_ID_METADATA_KEY: spend_log_session_id}
+                if spend_log_session_id
+                else metadata_from_headers
+            )
         return data
 
     @staticmethod
@@ -2417,7 +2453,7 @@ async def add_litellm_data_to_request(
     # them — from leaking into requester_metadata where guardrails and audit
     # paths may read from it.
     if "metadata" in data and isinstance(data["metadata"], dict):
-        data[_metadata_variable_name]["requester_metadata"] = copy.deepcopy(data["metadata"])
+        data[_metadata_variable_name]["requester_metadata"] = _requester_metadata_snapshot(data["metadata"])
         if _metadata_variable_name == "litellm_metadata":
             data[_metadata_variable_name].update(
                 _promoted_trace_control_fields(

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -7713,6 +7713,11 @@ async def block_key(
             code=status.HTTP_404_NOT_FOUND,
         )
 
+    record: Final = await _prisma_table(VerificationTokenRepository(prisma_client)).update(
+        where={"token": hashed_token},
+        data=with_settings_updated_at({"blocked": True}),
+    )
+
     if is_audit_logging_enabled():
         asyncio.create_task(
             create_audit_log_for_update(
@@ -7734,11 +7739,6 @@ async def block_key(
             )
         )
 
-    record: Final = await _prisma_table(VerificationTokenRepository(prisma_client)).update(
-        where={"token": hashed_token},
-        data=with_settings_updated_at({"blocked": True}),
-    )
-
     ## UPDATE KEY CACHE - invalidate so next read re-fetches from DB
     await delete_cache_key_object(
         hashed_token=hashed_token,
@@ -7827,6 +7827,11 @@ async def unblock_key(
             code=status.HTTP_404_NOT_FOUND,
         )
 
+    record: Final = await _prisma_table(VerificationTokenRepository(prisma_client)).update(
+        where={"token": hashed_token},
+        data=with_settings_updated_at({"blocked": False}),
+    )
+
     if is_audit_logging_enabled():
         asyncio.create_task(
             create_audit_log_for_update(
@@ -7848,11 +7853,6 @@ async def unblock_key(
             )
         )
 
-    record: Final = await _prisma_table(VerificationTokenRepository(prisma_client)).update(
-        where={"token": hashed_token},
-        data=with_settings_updated_at({"blocked": False}),
-    )
-
     ## UPDATE KEY CACHE - invalidate so next read re-fetches from DB
     await delete_cache_key_object(
         hashed_token=hashed_token,

litellm/proxy/management_helpers/audit_logs.py

@@ -5,7 +5,7 @@ Functions to create audit logs for LiteLLM Proxy
 import asyncio
 import json
 from datetime import datetime, timezone
-from typing import Final
+from typing import Final, TypeVar
 
 import litellm
 from litellm._logging import verbose_proxy_logger
@@ -134,6 +134,56 @@ def _audit_log_task_done_callback(task: asyncio.Task) -> None:
         verbose_proxy_logger.error("Audit log callback task failed: %s", exc, exc_info=exc)
 
 
+_T: Final = TypeVar("_T")
+
+
+class AuditTaskRegistry:
+    __slots__ = ("_pending_tasks",)
+
+    def __init__(self) -> None:
+        self._pending_tasks: Final[set[asyncio.Task[object]]] = set()  # mutable-ok: background task tracking
+
+    def track(self, task: asyncio.Task[_T]) -> asyncio.Task[_T]:
+        if task.done() or task in self._pending_tasks:
+            return task
+        self._pending_tasks.add(task)
+        task.add_done_callback(self._on_task_done)
+        return task
+
+    def _on_task_done(self, task: asyncio.Task[object]) -> None:
+        self._pending_tasks.discard(task)
+        _audit_log_task_done_callback(task)
+
+    async def drain(self, timeout: float = 5.0) -> None:
+        loop: Final = asyncio.get_running_loop()
+        deadline: Final = loop.time() + timeout
+        while self._pending_tasks:
+            _, pending = await asyncio.wait(tuple(self._pending_tasks), timeout=max(deadline - loop.time(), 0))
+            if pending:
+                self._pending_tasks.difference_update(pending)
+                verbose_proxy_logger.warning(
+                    "Timed out draining %d audit tasks on shutdown, dropping: %s",
+                    len(pending),
+                    ", ".join(sorted(task.get_name() for task in pending)),
+                )
+                return
+
+
+_audit_task_registry: Final = AuditTaskRegistry()
+
+
+def track_audit_task(task: asyncio.Task[_T]) -> asyncio.Task[_T]:
+    return _audit_task_registry.track(task)
+
+
+async def drain_audit_tasks(timeout: float = 5.0) -> None:
+    await _audit_task_registry.drain(timeout=timeout)
+
+
+def _audit_task_name(request_data: LiteLLM_AuditLogs, destination: str) -> str:
+    return f"{request_data.action} {request_data.table_name.value} {request_data.object_id} to {destination}"
+
+
 async def _dispatch_audit_log_to_callbacks(
     request_data: LiteLLM_AuditLogs,
 ) -> None:
@@ -153,8 +203,12 @@ async def _dispatch_audit_log_to_callbacks(
                     continue
 
             if isinstance(resolved, CustomLogger):
-                task = asyncio.create_task(resolved.async_log_audit_log_event(payload))
-                task.add_done_callback(_audit_log_task_done_callback)
+                track_audit_task(
+                    asyncio.create_task(
+                        resolved.async_log_audit_log_event(payload),
+                        name=_audit_task_name(request_data, type(resolved).__name__),
+                    )
+                )
         except Exception as e:
             verbose_proxy_logger.error("Failed dispatching audit log to callback: %s", e)
 
@@ -204,6 +258,14 @@ async def create_object_audit_log(
     )
 
 
+async def _insert_audit_log_row(repository: AuditLogRepository, request_data: LiteLLM_AuditLogs) -> None:
+    try:
+        await repository.table.create(data={**request_data.model_dump(exclude_none=True)})
+    except Exception as e:
+        # [Non-Blocking Exception. Do not allow blocking LLM API call]
+        verbose_proxy_logger.error("Failed Creating audit log %s", e)
+
+
 async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs):
     """
     Create an audit log for an object.
@@ -231,14 +293,9 @@ async def create_audit_log_for_update(request_data: LiteLLM_AuditLogs):
         verbose_proxy_logger.error("prisma_client is None, cannot write audit log to DB")
         return
 
-    _request_data: Final = request_data.model_dump(exclude_none=True)
-
-    try:
-        await AuditLogRepository(prisma_client).table.create(
-            data={
-                **_request_data,
-            }
+    await track_audit_task(
+        asyncio.create_task(
+            _insert_audit_log_row(AuditLogRepository(prisma_client), request_data),
+            name=_audit_task_name(request_data, "the database"),
         )
-    except Exception as e:
-        # [Non-Blocking Exception. Do not allow blocking LLM API call]
-        verbose_proxy_logger.error("Failed Creating audit log %s", e)
+    )

litellm/proxy/proxy_server.py

@@ -1200,6 +1200,20 @@ async def _flush_spend_logs_queue_on_shutdown() -> None:
         verbose_proxy_logger.exception("Error flushing spend logs queue on shutdown: %s", e)
 
 
+async def _drain_audit_tasks_on_shutdown() -> None:
+    try:
+        from litellm.proxy.management_helpers.audit_logs import drain_audit_tasks
+
+        await drain_audit_tasks()
+    except Exception as e:  # noqa: BLE001  # shutdown must continue even if the drain fails
+        verbose_proxy_logger.exception("Error draining audit tasks on shutdown: %s", e)
+
+
+async def _flush_request_accumulators(prisma: PrismaClient) -> None:
+    await flush_gateway_requests(prisma, gateway_request_accumulator)
+    await flush_request_errors(prisma, request_error_accumulator)
+
+
 async def proxy_shutdown_event(worker_heartbeat: ProxyWorkerHeartbeat | None = None) -> None:
     global prisma_client, master_key, user_custom_auth, user_custom_key_generate, user_custom_key_update
     verbose_proxy_logger.info("Shutting down LiteLLM Proxy Server")
@@ -1210,10 +1224,11 @@ async def proxy_shutdown_event(worker_heartbeat: ProxyWorkerHeartbeat | None = N
         # is lost, and a write attempted after disconnect raises
         # ClientNotConnectedError rather than persisting anything. Ordering this
         # inside the same guard is what keeps the two from drifting apart.
-        await flush_gateway_requests(prisma_client, gateway_request_accumulator)
-        await flush_request_errors(prisma_client, request_error_accumulator)
+        await asyncio.gather(_drain_audit_tasks_on_shutdown(), _flush_request_accumulators(prisma_client))
         verbose_proxy_logger.debug("Disconnecting from Prisma")
         await prisma_client.disconnect()
+    else:
+        await _drain_audit_tasks_on_shutdown()
 
     await drain_post_call_redis_batches()
     if litellm.cache is not None:
@@ -1509,7 +1524,6 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState
                     await asyncio.sleep(5)
 
         asyncio.create_task(_run_agent_grant_id_migration())
-        await install_background_interaction_settlement(prisma_client)
 
     ## A coordination_redis block saved from the admin UI lives in the database,
     ## which is only reachable once the prisma client exists. Apply it here, before
@@ -1636,6 +1650,10 @@ async def proxy_startup_event(app: FastAPI) -> AsyncGenerator[ProxyLifespanState
         ## SYNC UI SETTINGS ##
         await ProxyStartupEvent.sync_ui_settings_to_general_settings()
 
+        await install_background_interaction_settlement(
+            prisma_client, router=lambda: llm_router, spend_logs_kept=lambda: disable_spend_logs is False
+        )
+
     # Start background health checks AFTER models are loaded and index is built
     if use_background_health_checks:
         asyncio.create_task(_run_background_health_check())  # start the background health check coroutine.
@@ -14473,6 +14491,16 @@ async def _try_provider_token_count(
     return result
 
 
+_PROVIDERS_COUNTING_WITHOUT_MESSAGES: Final = frozenset(
+    {litellm.LlmProviders.GEMINI.value, litellm.LlmProviders.VERTEX_AI.value}
+)
+
+
+def _drops_params(deployment: Mapping[str, object] | None) -> bool:
+    litellm_params: Final = as_str_mapping(None if deployment is None else deployment.get("litellm_params"))
+    return litellm.drop_params is True or (litellm_params is not None and litellm_params.get("drop_params") is True)
+
+
 def _system_message(system: object) -> ChatCompletionSystemMessage | None:
     if not isinstance(system, (str, list)) or not system:
         return None
@@ -14539,8 +14567,8 @@ async def count_request_tokens(
             litellm_model_name = litellm_model_name.split("/", 1)[1]
 
     model_to_use: str = (
-        litellm_model_name or request.model
-    )  # use litellm model name, if it's not avalable then fallback to request.model
+        litellm_model_name if isinstance(litellm_model_name, str) and litellm_model_name else request.model
+    )
 
     # Try provider-specific token counting first - only for non-direct requests (from provider endpoints)
     provider_counter: BaseTokenCounter | None = None
@@ -14551,13 +14579,23 @@ async def count_request_tokens(
         if _model is not None:
             model_to_use = _model
 
+    conversation_empty: Final = not messages and not contents
+    if prompt and not conversation_empty and not _drops_params(deployment):
+        raise ProxyException(
+            message="prompt cannot be sent with messages or contents. Set drop_params to count them without the prompt",
+            type="invalid_request_error",
+            param="prompt",
+            code=400,
+        )
+    prompt_only: Final = bool(prompt) and conversation_empty
     if provider_counter is not None:
+        prompt_as_message: Final = prompt_only and custom_llm_provider in _PROVIDERS_COUNTING_WITHOUT_MESSAGES
         result: Final = await _try_provider_token_count(
             provider_counter=provider_counter,
             custom_llm_provider=custom_llm_provider,
             model_to_use=model_to_use,
-            messages=messages,
-            contents=contents,
+            messages=[{"role": "user", "content": prompt}] if prompt_as_message else messages,
+            contents=None if prompt_as_message else contents,
             deployment=deployment,
             request_model=request.model,
             tools=tools,
@@ -14590,8 +14628,10 @@ async def count_request_tokens(
 
     tokenizer_used: Final = str(_tokenizer_used["type"])
     system_message: Final = _system_message(system)
+    prompt_turn: Final = prompt_only and not (system_message is None and not tools)
     typed_messages: Final = cast(  # cast-ok: request messages are raw chat-shaped dicts that token_counter normalizes
-        Sequence[AllMessageValues] | None, messages
+        Sequence[AllMessageValues] | None,
+        [{"role": "user", "content": prompt}] if prompt_turn else (None if prompt_only else messages),
     )
     counted_messages: Final = (
         typed_messages if typed_messages is None or system_message is None else (system_message, *typed_messages)
@@ -14601,7 +14641,7 @@ async def count_request_tokens(
     )
     total_tokens: Final = await offload_token_count(litellm.token_counter)(
         model=model_to_use,
-        text=prompt,
+        text=prompt if counted_messages is None else None,
         messages=counted_messages,
         tools=counted_tools,
         custom_tokenizer=_tokenizer_used,
@@ -19658,6 +19698,26 @@ async def delete_config_general_settings(
     return response
 
 
+def _configured_callback_names(value: object) -> tuple[JsonValue, ...]:
+    if isinstance(value, str):
+        return (value,)
+    if isinstance(value, list):
+        return tuple(cast(Iterable[JsonValue], value))  # cast-ok: config.yaml values are untyped
+    return ()
+
+
+def _is_callback_name(entry: object, callback_name: str) -> bool:
+    return isinstance(entry, str) and entry.lower() == callback_name
+
+
+def _callback_lists_to_rewrite(
+    matching_keys: tuple[str, ...], owned_by_config: Callable[[str], bool]
+) -> tuple[str, ...]:
+    """The matching lists the database owns, or all of them when the config file owns every one."""
+    database_owned: Final = tuple(key for key in matching_keys if not owned_by_config(key))
+    return database_owned or matching_keys
+
+
 @router.post(
     "/config/callback/delete",
     tags=["config.yaml"],
@@ -19693,24 +19753,42 @@ async def delete_callback(
 
     try:
         # Get current configuration
-        config: Final = await proxy_config.get_config()
+        config: Final = cast(  # cast-ok: get_config returns an untyped config dict
+            dict[str, object],
+            await proxy_config.get_config(),
+        )
         callback_name: Final = data.callback_name.lower()
 
         # Check if callback exists in current configuration
-        litellm_settings: Final = config.get("litellm_settings", {})
-        success_callbacks: Final = litellm_settings.get("success_callback", [])
+        litellm_settings: Final[Mapping[str, object]] = cast(  # cast-ok: the config dict's values are untyped
+            Mapping[str, object],
+            config.get("litellm_settings", {}),
+        )
+        configured_lists: Final = {
+            key: _configured_callback_names(litellm_settings.get(key)) for key in _DB_CONFIG_CALLBACK_EVENT_TYPES
+        }
+        matching_keys: Final = tuple(
+            key
+            for key, names in configured_lists.items()
+            if any(_is_callback_name(entry, callback_name) for entry in names)
+        )
 
-        if callback_name not in success_callbacks:
+        if not matching_keys:
             raise HTTPException(
                 status_code=404,
                 detail={"error": f"Callback '{callback_name}' not found in active configuration"},
             )
 
-        before_success_callbacks: Final = list(success_callbacks)
-
-        # Remove callback from success_callback list
-        success_callbacks.remove(callback_name)
-        config.setdefault("litellm_settings", {})["success_callback"] = success_callbacks
+        rewritten_keys: Final = _callback_lists_to_rewrite(matching_keys, proxy_config.litellm_settings.owned_by_config)
+        remaining_lists: Final = MappingProxyType(
+            {
+                key: tuple(entry for entry in configured_lists[key] if not _is_callback_name(entry, callback_name))
+                for key in rewritten_keys
+            }
+        )
+        before_callbacks: Final[JsonValue] = {key: list(configured_lists[key]) for key in rewritten_keys}
+        after_callbacks: Final[JsonValue] = {key: list(names) for key, names in remaining_lists.items()}
+        config["litellm_settings"] = {**litellm_settings, **after_callbacks}
 
         # Save the updated configuration
         await proxy_config.save_config(new_config=config)
@@ -19719,8 +19797,8 @@ async def delete_callback(
             create_config_audit_log(
                 "litellm_settings",
                 "deleted",
-                {"success_callback": before_success_callbacks},
-                {"success_callback": success_callbacks},
+                before_callbacks,
+                after_callbacks,
                 user_api_key_dict,
             )
         )
@@ -19731,7 +19809,7 @@ async def delete_callback(
         return {
             "message": f"Successfully deleted callback: {callback_name}",
             "removed_callback": callback_name,
-            "remaining_callbacks": success_callbacks,
+            "remaining_callbacks": tuple(chain.from_iterable(remaining_lists.values())),
             "deleted_at": datetime.now().isoformat(),
         }
 

litellm/proxy/schema.prisma

@@ -1981,6 +1981,7 @@ model LiteLLM_BackgroundInteractionSettlement {
   outcome             String?
 
   @@index([claimed_at], map: "idx_background_interaction_settlement_claimed_at")
+  @@index([settled_at], map: "idx_background_interaction_settlement_settled_at")
 }
 
 model LiteLLM_Lens {

litellm/proxy/spend_tracking/background_interaction_settlement.py

@@ -1,21 +1,25 @@
 import asyncio
 import os
 import socket
-from collections.abc import Awaitable, Mapping, Sequence
-from dataclasses import dataclass
-from datetime import datetime, timezone
+from collections.abc import Awaitable, Callable, Mapping, Sequence
+from dataclasses import dataclass, replace
+from datetime import datetime, timedelta, timezone
 from itertools import chain
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Final, Protocol, TypeVar
+from typing import TYPE_CHECKING, Final, Literal, Protocol, TypeVar
 
 from pydantic import ValidationError
 from typing_extensions import ReadOnly, TypedDict
 
 from litellm._logging import verbose_proxy_logger
-from litellm.constants import BACKGROUND_INTERACTION_COST_POLLING_ENABLED
+from litellm.constants import (
+    BACKGROUND_INTERACTION_COST_POLLING_ENABLED,
+    BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS,
+)
 from litellm.interactions.background_cost_polling import (
     DEFAULT_POLL_SCHEDULE,
     BackgroundInteractionCreateContext,
+    BackgroundInteractionPollContext,
     FetchInteraction,
     PendingBackgroundInteraction,
     PollSchedule,
@@ -24,10 +28,13 @@ from litellm.interactions.background_cost_polling import (
     fetch_background_interaction,
     resume_unsettled_background_interactions,
 )
-from litellm.repositories.table_repositories import BackgroundInteractionSettlementRepository
+from litellm.repositories.table_repositories import BackgroundInteractionSettlementRepository, SpendLogsRepository
+from litellm.types.interactions import InteractionsAPIResponse
+from litellm.types.router import CredentialLiteLLMParams
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient
+    from litellm.router import Router
 
 
 class _SettlementRow(Protocol):
@@ -41,6 +48,8 @@ class _SettlementRow(Protocol):
     def created_at(self) -> datetime: ...
     @property
     def claimed_at(self) -> datetime | None: ...
+    @property
+    def settled_at(self) -> datetime | None: ...
 
 
 class _NewSettlementRow(TypedDict):
@@ -59,8 +68,24 @@ class _UnclaimedRowKey(TypedDict):
     claimed_at: ReadOnly[None]
 
 
-class _UnclaimedRows(TypedDict):
-    claimed_at: ReadOnly[None]
+class _Before(TypedDict):
+    lt: ReadOnly[datetime]
+
+
+class _ExpiredClaimRowKey(TypedDict):
+    interaction_id: ReadOnly[str]
+    settled_at: ReadOnly[None]
+    claimed_at: ReadOnly[_Before]
+
+
+class _OwnClaimRowKey(TypedDict):
+    interaction_id: ReadOnly[str]
+    claimed_by: ReadOnly[str]
+    settled_at: ReadOnly[None]
+
+
+class _UnsettledRows(TypedDict):
+    settled_at: ReadOnly[None]
 
 
 class _Claim(TypedDict):
@@ -79,9 +104,19 @@ class _SettlementTableActions(Protocol):
 
     def find_unique(self, *, where: _RowKey) -> Awaitable[_SettlementRow | None]: ...
 
-    def find_many(self, *, where: _UnclaimedRows) -> Awaitable[Sequence[_SettlementRow]]: ...
+    def find_many(self, *, where: _UnsettledRows) -> Awaitable[Sequence[_SettlementRow]]: ...
 
-    def update_many(self, *, data: _Claim | _Outcome, where: _RowKey | _UnclaimedRowKey) -> Awaitable[int]: ...
+    def update_many(
+        self, *, data: _Claim | _Outcome, where: _RowKey | _UnclaimedRowKey | _ExpiredClaimRowKey | _OwnClaimRowKey
+    ) -> Awaitable[int]: ...
+
+
+class _SpendLogKey(TypedDict):
+    request_id: ReadOnly[str]
+
+
+class _SpendLogLookup(Protocol):
+    def find_unique(self, *, where: _SpendLogKey) -> Awaitable[object | None]: ...
 
 
 _CLEARED_CREATE_CONTEXT: Final[Mapping[str, object]] = MappingProxyType({})
@@ -103,11 +138,7 @@ def _json(data: Mapping[str, object]) -> object:
     return Json.keys(**data)
 
 
-def _pending_rows(rows: Sequence[_SettlementRow]) -> tuple[PendingBackgroundInteraction, ...]:
-    return tuple(chain.from_iterable(_pending_row(row) for row in rows))
-
-
-def _pending_row(row: _SettlementRow) -> tuple[PendingBackgroundInteraction, ...]:
+def _pending_row(row: _SettlementRow, claim_expires_at: datetime | None) -> tuple[PendingBackgroundInteraction, ...]:
     try:
         create_context: Final = BackgroundInteractionCreateContext.model_validate(row.create_context)
     except ValidationError:
@@ -122,14 +153,31 @@ def _pending_row(row: _SettlementRow) -> tuple[PendingBackgroundInteraction, ...
             custom_llm_provider=row.custom_llm_provider,
             create_context=create_context,
             created_at=row.created_at,
+            claim_expires_at=claim_expires_at,
         ),
     )
 
 
 @dataclass(frozen=True, slots=True)
 class PrismaBackgroundSettlementStore:
+    """
+    A claim is a lease, not a settlement: the settler that claims a row bills
+    or releases it and then records the outcome, so a row claimed longer ago
+    than the lease with no outcome belongs to a settler that died in between,
+    and any settler may claim it again. A live settler renews its claim while
+    it settles, so it is never overtaken. The settler before may have billed
+    and then died or failed to record it, so a takeover bills only when the
+    interaction has no spend row yet, and otherwise records it as billed. The
+    spend row is read before the claim is taken, so a failed read takes
+    nothing and the claim is retried. With spend logs off there is no row to
+    tell, so a takeover bills nothing rather than risk billing twice.
+    """
+
     table: _SettlementTableActions
+    spend_logs: _SpendLogLookup
+    spend_logs_kept: Callable[[], bool]
     claimed_by: str
+    claim_lease_seconds: float = BACKGROUND_INTERACTION_SETTLEMENT_CLAIM_LEASE_SECONDS
 
     async def register(self, pending: PendingBackgroundInteraction) -> None:
         await self.table.create(
@@ -143,24 +191,71 @@ class PrismaBackgroundSettlementStore:
 
     async def pending(self, interaction_id: str) -> PendingBackgroundInteraction | None:
         row: Final = await self._row(interaction_id)
-        if row is None or row.claimed_at is not None:
+        if row is None or self._is_owned(row):
             return None
-        return next(iter(_pending_row(row)), None)
+        return next(iter(_pending_row(row, claim_expires_at=None)), None)
 
     async def is_claimed(self, interaction_id: str) -> bool:
         row: Final = await self._row(interaction_id)
-        return row is not None and row.claimed_at is not None
+        return row is not None and self._is_owned(row)
 
     async def claim(self, interaction_id: str) -> bool:
+        if await self._claim_where(_UnclaimedRowKey(interaction_id=interaction_id, claimed_at=None)):
+            return True
+        settled_before: Final = await self._outcome_a_dead_settler_left(interaction_id)
+        took_over: Final = await self._claim_where(
+            _ExpiredClaimRowKey(
+                interaction_id=interaction_id,
+                settled_at=None,
+                claimed_at=_Before(lt=datetime.now(timezone.utc) - self._lease),
+            )
+        )
+        if not took_over or settled_before is None:
+            return took_over
+        if settled_before == "unsettled":
+            verbose_proxy_logger.warning(
+                "Background interaction %s lost its settler mid-settle, and with spend logs off nothing shows "
+                "whether it was billed, so it is left unbilled rather than risk billing it twice",
+                interaction_id,
+            )
+        await self.record_outcome(interaction_id, settled_before)
+        return False
+
+    async def renew_claim(self, interaction_id: str) -> None:
+        await self.table.update_many(
+            data=_Claim(claimed_at=datetime.now(timezone.utc), claimed_by=self.claimed_by),
+            where=_OwnClaimRowKey(interaction_id=interaction_id, claimed_by=self.claimed_by, settled_at=None),
+        )
+
+    async def _claim_where(self, where: _UnclaimedRowKey | _ExpiredClaimRowKey) -> bool:
         claimed_rows: Final = await _read_from_a_table_that_may_not_exist(
             self.table.update_many(
-                data=_Claim(claimed_at=datetime.now(timezone.utc), claimed_by=self.claimed_by),
-                where=_UnclaimedRowKey(interaction_id=interaction_id, claimed_at=None),
+                data=_Claim(claimed_at=datetime.now(timezone.utc), claimed_by=self.claimed_by), where=where
             ),
             when_missing=0,
         )
         return claimed_rows == 1
 
+    async def _outcome_a_dead_settler_left(self, interaction_id: str) -> Literal["billed", "unsettled"] | None:
+        if not self.spend_logs_kept():
+            return "unsettled"
+        if await self.spend_logs.find_unique(where=_SpendLogKey(request_id=interaction_id)) is None:
+            return None
+        return "billed"
+
+    @property
+    def _lease(self) -> timedelta:
+        return timedelta(seconds=self.claim_lease_seconds)
+
+    def _claim_expires_at(self, row: _SettlementRow) -> datetime | None:
+        return None if row.claimed_at is None else row.claimed_at + self._lease
+
+    def _is_owned(self, row: _SettlementRow) -> bool:
+        if row.settled_at is not None:
+            return True
+        claim_expires_at: Final = self._claim_expires_at(row)
+        return claim_expires_at is not None and claim_expires_at > datetime.now(timezone.utc)
+
     async def _row(self, interaction_id: str) -> _SettlementRow | None:
         return await _read_from_a_table_that_may_not_exist(
             self.table.find_unique(where=_RowKey(interaction_id=interaction_id)), when_missing=None
@@ -174,19 +269,78 @@ class PrismaBackgroundSettlementStore:
             where=_RowKey(interaction_id=interaction_id),
         )
 
-    async def unclaimed(self) -> Sequence[PendingBackgroundInteraction]:
-        return _pending_rows(await self.table.find_many(where=_UnclaimedRows(claimed_at=None)))
+    async def unsettled(self) -> Sequence[PendingBackgroundInteraction]:
+        rows: Final = await self.table.find_many(where=_UnsettledRows(settled_at=None))
+        return tuple(chain.from_iterable(_pending_row(row, self._claim_expires_at(row)) for row in rows))
+
+
+def _provider_credentials(router: "Router", deployment_id: str, custom_llm_provider: str) -> Mapping[str, object]:
+    credentials: Final = router.get_deployment_credentials_with_provider(model_id=deployment_id)
+    if credentials is None or credentials.get("custom_llm_provider") != custom_llm_provider:
+        return MappingProxyType({})
+    return MappingProxyType(credentials)
+
+
+def _group_deployment_credentials(router: "Router", context: BackgroundInteractionPollContext) -> Mapping[str, object]:
+    if context.model_group is None:
+        return MappingProxyType({})
+    candidates: Final = tuple(
+        credentials
+        for deployment_id in router.get_model_ids(model_name=context.model_group)
+        if (credentials := _provider_credentials(router, deployment_id, context.custom_llm_provider))
+    )
+    return candidates[0] if len(candidates) == 1 else MappingProxyType({})
+
+
+def _deployment_credentials(
+    router: "Router | None", context: BackgroundInteractionPollContext
+) -> CredentialLiteLLMParams | None:
+    if router is None:
+        return None
+    serving: Final = (
+        _provider_credentials(router, context.deployment_id, context.custom_llm_provider)
+        if context.deployment_id is not None and router.get_deployment(model_id=context.deployment_id) is not None
+        else _group_deployment_credentials(router, context)
+    )
+    return CredentialLiteLLMParams.model_validate(serving) if serving else None
+
+
+def fetch_with_deployment_credentials(
+    router: Callable[[], "Router | None"], fetch_interaction: FetchInteraction = fetch_background_interaction
+) -> FetchInteraction:
+    """
+    A resumed poll has no request to take credentials from, and they are never
+    stored, so it fetches with the credentials of the deployment that served
+    the create, looked up by its id on the live router at each fetch so a
+    deployment loaded from the database after startup is found too. Editing a
+    deployment changes its id, so a missing id falls back to the create's model
+    group when exactly one of its deployments serves the same provider. Anything
+    else falls back to the provider's environment credentials, never to a
+    wildcard route, whose key belongs to whatever provider it routes to.
+    """
+
+    async def fetch(context: BackgroundInteractionPollContext) -> InteractionsAPIResponse:
+        credentials: Final = _deployment_credentials(router(), context)
+        if credentials is None:
+            return await fetch_interaction(context)
+        return await fetch_interaction(replace(context, credentials=credentials))
+
+    return fetch
 
 
 async def configure_background_interaction_settlement(
     table: _SettlementTableActions,
+    spend_logs: _SpendLogLookup,
+    spend_logs_kept: Callable[[], bool],
     claimed_by: str,
     fetch_interaction: FetchInteraction = fetch_background_interaction,
     schedule: PollSchedule = DEFAULT_POLL_SCHEDULE,
 ) -> tuple["asyncio.Task[SettlementOutcome | None]", ...]:
     if not BACKGROUND_INTERACTION_COST_POLLING_ENABLED:
         return ()
-    store: Final = PrismaBackgroundSettlementStore(table=table, claimed_by=claimed_by)
+    store: Final = PrismaBackgroundSettlementStore(
+        table=table, spend_logs=spend_logs, spend_logs_kept=spend_logs_kept, claimed_by=claimed_by
+    )
     configure_background_settlement_store(store)
     resumed: Final = await resume_unsettled_background_interactions(store, fetch_interaction, schedule)
     if resumed:
@@ -194,11 +348,16 @@ async def configure_background_interaction_settlement(
     return resumed
 
 
-async def install_background_interaction_settlement(prisma_client: "PrismaClient") -> None:
+async def install_background_interaction_settlement(
+    prisma_client: "PrismaClient", router: Callable[[], "Router | None"], spend_logs_kept: Callable[[], bool]
+) -> None:
     try:
         await configure_background_interaction_settlement(
             table=BackgroundInteractionSettlementRepository(prisma_client).table,
+            spend_logs=SpendLogsRepository(prisma_client).table,
+            spend_logs_kept=spend_logs_kept,
             claimed_by=f"{socket.gethostname()}:{os.getpid()}",
+            fetch_interaction=fetch_with_deployment_credentials(router),
         )
     except Exception as e:  # noqa: BLE001  # a boot step must survive any DB error; billing then settles in-process as before
         verbose_proxy_logger.warning(

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -15,6 +15,7 @@ from pydantic import BaseModel, JsonValue, TypeAdapter, ValidationError
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import (
+    CALLER_SESSION_ID_METADATA_KEY,
     CLI_SESSION_KEY_PREFIX,
     EMPTY_MAPPING,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
@@ -33,6 +34,8 @@ from litellm.constants import (
 from litellm.constants import (
     MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB,
 )
+from litellm.integrations.otel.model.metadata import caller_session_id
+from litellm.integrations.otel.model.trace_controls import TraceControls
 from litellm.litellm_core_utils.classifier_logging import (
     CLASSIFIER_AUDIT_FIELDS,
     classifier_audit_fields,
@@ -570,11 +573,11 @@ def get_logging_payload(
         elif isinstance(_usage, dict):
             usage = _usage
 
-    # A request that failed mid-stream has no usable response_obj usage, but the
-    # streaming handler may have recovered the usage from the chunks already
-    # delivered. Honor that override so the partial usage lands in spend tracking.
+    # A request that failed mid-stream has no usable response_obj usage, and a stream
+    # continued on a fallback only has the fallback's. combined_usage_object holds the
+    # usage of every call the client's answer took, so it wins in both cases.
     _combined_usage: Final = kwargs.get("combined_usage_object")
-    if not usage and isinstance(_combined_usage, litellm.Usage):
+    if (not usage or "continued_partial_usage" in kwargs) and isinstance(_combined_usage, litellm.Usage):
         usage = _combined_usage.model_dump()
 
     id = get_spend_logs_id(call_type or "acompletion", response_obj_dict, kwargs, metadata)
@@ -930,6 +933,16 @@ def _get_batch_trace_session_id(call_type: str | None, request_id: str | None) -
     return request_id.removesuffix(BATCH_COST_REQUEST_ID_SUFFIX)
 
 
+def _caller_session_for_spend_log(kwargs: Mapping[str, object], metadata: Mapping[str, object] | None) -> str | None:
+    beside_trace_header: Final = metadata.get(CALLER_SESSION_ID_METADATA_KEY) if metadata else None
+    caller_session: Final = (
+        beside_trace_header
+        if isinstance(beside_trace_header, str) and beside_trace_header
+        else caller_session_id(kwargs, TraceControls())
+    )
+    return caller_session if caller_session and "\x00" not in caller_session else None
+
+
 def _get_session_id_for_spend_log(
     kwargs: Mapping[str, object],
     metadata: Mapping[str, object] | None,
@@ -937,9 +950,12 @@ def _get_session_id_for_spend_log(
     omit_when_missing: bool,
     batch_trace_session_id: str | None = None,
 ) -> str | None:
-    """Under `omit` only `metadata.session_id`, the key Langfuse reads, counts as a session; `litellm_session_id` may
-    be a copied trace id. Batch call types carry a deterministic session derived from the batch id, which outranks
-    the per-request trace ids because those differ between the create call and the cost poller's row."""
+    """Resolve the session id for the spend log row: `omit` honors only metadata.session_id,
+    batch sessions outrank everything, then the conversation id the caller sent (also one an x-litellm-trace-id header
+    displaced from the request's session), then trace ids.
+
+    A session minted by `missing_session_id: generate` or copied from the request's trace id is not a conversation, so
+    it yields to the trace id, which the Responses bridge points at the previous turn's session to keep a chain whole."""
     if omit_when_missing:
         session_id: Final = metadata.get("session_id") if metadata else None
         return str(session_id) if session_id else None
@@ -948,11 +964,14 @@ def _get_session_id_for_spend_log(
 
     if batch_trace_session_id is not None:
         return batch_trace_session_id
-    if standard_logging_payload is not None and standard_logging_payload.get("trace_id") is not None:
-        return str(standard_logging_payload.get("trace_id"))
-    if kwargs.get("litellm_trace_id") is not None:
-        return str(kwargs.get("litellm_trace_id"))
-    return str(uuid.uuid4())
+    if litellm.request_correlation_in_logs and (caller_session := _caller_session_for_spend_log(kwargs, metadata)):
+        return caller_session
+    match standard_logging_payload:
+        case {"trace_id": trace_id} if trace_id is not None:
+            return str(trace_id)
+        case _:
+            litellm_trace_id: Final = kwargs.get("litellm_trace_id")
+            return str(litellm_trace_id) if litellm_trace_id is not None else str(uuid.uuid4())
 
 
 def _get_request_duration_ms(start_time: datetime, end_time: datetime) -> int | None:

litellm/proxy/ui_crud_endpoints/proxy_setting_endpoints.py

@@ -343,7 +343,7 @@ class UISettings(LiteLLMBaseModel):
     )
 
     enable_chat_ui: bool = Field(
-        default=False,
+        default=True,
         description="If true, shows the Chat page in the UI sidebar, letting users chat with an LLM and connect their own MCP server credentials via OAuth.",
     )
 

litellm/realtime_api/main.py

@@ -500,11 +500,13 @@ async def _arealtime(
         )
         return
 
+    drop_params: Final = litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True
     provider_config: BaseRealtimeConfig | None = None
     if _custom_llm_provider in LlmProviders._member_map_.values():
         provider_config = ProviderConfigManager.get_provider_realtime_config(
             model=model,
             provider=LlmProviders(_custom_llm_provider),
+            drop_params=drop_params,
         )
     if provider_config is not None:
         await base_llm_http_handler.async_realtime(
@@ -585,7 +587,6 @@ async def _arealtime(
         aws_bedrock_runtime_endpoint: Final = kwargs.get("aws_bedrock_runtime_endpoint")
         aws_external_id: Final = kwargs.get("aws_external_id")
         aws_session_tags: Final = kwargs.get("aws_session_tags")
-        drop_params: Final = litellm.drop_params is True or normalize_drop_params(kwargs.get("drop_params")) is True
 
         await bedrock_realtime.async_realtime(
             model=model,
@@ -671,6 +672,7 @@ async def _arealtime(
             resolve_access_token=resolve_vertex_access_token,
             project=resolved_project,
             location=vertex_location,
+            drop_params=drop_params,
         )
 
         await base_llm_http_handler.async_realtime(

litellm/responses/bridge_check.py

@@ -4,6 +4,7 @@ from urllib.parse import urlsplit
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 from litellm.llms.azure_ai.common_utils import (
     azure_ai_supports_native_responses,
     foundry_chat_rejects_function_tools_while_reasoning,
@@ -11,7 +12,8 @@ from litellm.llms.azure_ai.common_utils import (
 from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import OpenAIWebSearchOptions
-from litellm.utils import get_model_info_helper
+from litellm.types.utils import LlmProviders
+from litellm.utils import ProviderConfigManager, get_model_info_helper
 
 _OPENAI_DEFAULT_API_BASE: Final = "https://api.openai.com/v1"
 _OPENAI_API_HOST: Final = "api.openai.com"
@@ -36,6 +38,18 @@ def _resolve_openai_api_base(api_base: str | None) -> str:
     )
 
 
+def chat_target_accepts_reasoning_summary(model: str, custom_llm_provider: str | None) -> bool:
+    try:
+        local_model, provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
+        chat_config: Final = ProviderConfigManager.get_provider_chat_config(
+            model=local_model, provider=LlmProviders(provider)
+        )
+    except Exception as e:  # noqa: BLE001  # a capability probe must never fail the request it probes for
+        verbose_logger.debug("chat target reasoning summary check failed: %s", e)
+        return False
+    return chat_config is not None and chat_config.accepts_reasoning_summary_in_reasoning_effort
+
+
 def responses_api_bridge_check(
     model: str,
     custom_llm_provider: str,

litellm/responses/litellm_completion_transformation/session_handler.py

@@ -281,7 +281,13 @@ class ResponsesSessionHandler:
 
         SQL query
 
-        SELECT session_id FROM spend_logs WHERE response_id = previous_response_id, SELECT * FROM spend_logs WHERE session_id = session_id
+        SELECT session_id, api_key, "user", team_id FROM spend_logs WHERE request_id = previous_response_id, then
+        SELECT * FROM spend_logs with that session_id and the same owner
+
+        Only rows with the referenced response's owner are returned: the same user and team, and for a key with
+        neither, the same key. The session id can be caller supplied, so matching on it alone would mix owners. A
+        user's or team's regenerated or second key keeps the history; a key with neither does not, and turns written
+        by another owner (an admin continuing the chain) are not replayed.
 
         A just-finished turn gets a short second chance: the worker that served it may
         still be writing its spend log when the follow-up arrives, and an empty result
@@ -303,13 +309,24 @@ class ResponsesSessionHandler:
 
         query: Final = """
             WITH matching_session AS (
-                SELECT session_id
+                SELECT session_id, api_key, "user", team_id
                 FROM "LiteLLM_SpendLogs"
                 WHERE request_id = $1
             )
             SELECT *
-            FROM "LiteLLM_SpendLogs"
-            WHERE session_id IN (SELECT session_id FROM matching_session)
+            FROM "LiteLLM_SpendLogs" AS logs
+            WHERE EXISTS (
+                SELECT 1
+                FROM matching_session
+                WHERE matching_session.session_id = logs.session_id
+                  AND matching_session."user" IS NOT DISTINCT FROM logs."user"
+                  AND matching_session.team_id IS NOT DISTINCT FROM logs.team_id
+                  AND (
+                      COALESCE(matching_session."user", '') <> ''
+                      OR COALESCE(matching_session.team_id, '') <> ''
+                      OR matching_session.api_key = logs.api_key
+                  )
+            )
             ORDER BY "endTime" ASC;
         """
 

litellm/responses/litellm_completion_transformation/transformation.py

@@ -39,7 +39,11 @@ from typing_extensions import ReadOnly, TypedDict
 import litellm
 from litellm._logging import verbose_logger
 from litellm.caching import InMemoryCache
-from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER
+from litellm.constants import (
+    CALLER_SESSION_ID_METADATA_KEY,
+    REDACTED_BY_LITELLM,
+    REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER,
+)
 from litellm.integrations.anthropic_cache_control_hook import MAX_CACHE_CONTROL_BLOCKS, AnthropicCacheControlHook
 from litellm.litellm_core_utils.core_helpers import normalize_drop_params
 from litellm.litellm_core_utils.get_supported_openai_params import (
@@ -56,6 +60,7 @@ from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.llms.bedrock.common_utils import bedrock_declared_cache_point_support
 from litellm.llms.openrouter.chat.transformation import openrouter_model_supports_cache_control
 from litellm.llms.snowflake.chat.transformation import is_snowflake_claude_model
+from litellm.responses.bridge_check import chat_target_accepts_reasoning_summary
 from litellm.responses.litellm_completion_transformation.session_handler import (
     ResponsesSessionHandler,
 )
@@ -562,6 +567,14 @@ class ChatCompletionSession(TypedDict, total=False):
 ########### End of Initialize Classes used for Responses API  ###########
 
 
+def _chain_session_metadata(chain_session: str, request_kwargs: Mapping[str, object]) -> Mapping[str, object]:
+    key: Final = "litellm_metadata" if request_kwargs.get("litellm_metadata") else "metadata"
+    metadata: Final = request_kwargs.get(key)
+    if not isinstance(metadata, dict):
+        return MappingProxyType({})
+    return {key: {**_STR_KEY_DICT_ADAPTER.validate_python(metadata), CALLER_SESSION_ID_METADATA_KEY: chain_session}}
+
+
 class LiteLLMCompletionResponsesConfig:
     @staticmethod
     def get_supported_openai_params(model: str) -> list:
@@ -785,8 +798,8 @@ class LiteLLMCompletionResponsesConfig:
         provider either rejects it or silently drops it, and dropping it turns reasoning off while
         still billing for the turn. ``summary`` has no chat equivalent, so it rides the
         ``reasoning_summary`` alias, which ``litellm.completion`` reassembles into ``{effort,
-        summary}`` when it bridges the model back onto the Responses API, and is sent to nothing
-        else.
+        summary}`` when it bridges the model back onto the Responses API or targets a chat provider
+        whose config maps a summary ask (Gemini's thought summaries), and is sent to nothing else.
         """
         if not reasoning_param:
             return ResponsesReasoningChatForm(effort=None, summary=None)
@@ -807,7 +820,10 @@ class LiteLLMCompletionResponsesConfig:
             reasoning_summary=summary,
             api_base=api_base,
         )
-        return ResponsesReasoningChatForm(effort=effort, summary=summary if bridges_back else None)
+        forwards_summary: Final = bridges_back or (
+            isinstance(effort, str) and chat_target_accepts_reasoning_summary(model, custom_llm_provider)
+        )
+        return ResponsesReasoningChatForm(effort=effort, summary=summary if forwards_summary else None)
 
     @staticmethod
     def transform_responses_api_request_to_chat_completion_request(
@@ -1074,8 +1090,6 @@ class LiteLLMCompletionResponsesConfig:
             else:
                 # Both are empty - this likely means function_call_output had empty/invalid call_id
                 # Provide a helpful error message
-                import litellm
-
                 raise litellm.BadRequestError(
                     message=(
                         f"Unable to create messages for completion request. "
@@ -1090,8 +1104,14 @@ class LiteLLMCompletionResponsesConfig:
                 )
 
         litellm_completion_request["messages"] = combined_messages
-        litellm_completion_request["litellm_trace_id"] = chat_completion_session.get("litellm_session_id")
-        return litellm_completion_request
+        chain_session: Final = chat_completion_session.get("litellm_session_id")
+        litellm_completion_request["litellm_trace_id"] = chain_session
+        chain_session_metadata: Final[Mapping[str, object]] = (
+            _chain_session_metadata(chain_session, request_kwargs)
+            if chain_session and litellm.request_correlation_in_logs
+            else MappingProxyType({})
+        )
+        return {**litellm_completion_request, **chain_session_metadata}
 
     @staticmethod
     def _transform_response_input_param_to_chat_completion_message(

litellm/responses/streaming_iterator.py

@@ -31,6 +31,7 @@ from litellm.litellm_core_utils.core_helpers import process_response_headers
 from litellm.litellm_core_utils.internal_call_metadata import without_budget_reservation
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.llm_response_utils.get_api_base import get_api_base
+from litellm.litellm_core_utils.llm_response_utils.get_headers import provider_request_id_headers
 from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
     update_response_metadata,
 )
@@ -262,7 +263,9 @@ def stream_error_status_and_message(error_obj: object) -> tuple[int, str]:
     return _status_code_for_error_fields(error_type, error_code), message
 
 
-def _map_stream_error_to_exception(error_obj: object, model: str, custom_llm_provider: str) -> Exception:
+def _map_stream_error_to_exception(
+    error_obj: object, model: str, custom_llm_provider: str, headers: httpx.Headers | None = None
+) -> Exception:
     from litellm.llms.base_llm.chat.transformation import BaseLLMException
 
     error_message, error_type, error_code = _error_event_fields(error_obj)
@@ -272,6 +275,7 @@ def _map_stream_error_to_exception(error_obj: object, model: str, custom_llm_pro
         status_code=status_code,
         message=f"Error code: {status_code} - {{'error': {error_body}}}",
         body=error_body,
+        headers=headers,
     )
     try:
         return litellm.exception_type(
@@ -667,7 +671,12 @@ class BaseResponsesAPIStreamingIterator:
         )
 
     def _map_error_event_exception(self, error_obj: object) -> Exception:
-        return _map_stream_error_to_exception(error_obj, self.model or "", self.custom_llm_provider or "")
+        return _map_stream_error_to_exception(
+            error_obj,
+            self.model or "",
+            self.custom_llm_provider or "",
+            headers=provider_request_id_headers(self._raw_response_headers),
+        )
 
     def _maybe_raise_for_error_event(self, result: object) -> None:
         chunk_type: Final = getattr(result, "type", None)
@@ -683,11 +692,17 @@ class BaseResponsesAPIStreamingIterator:
         mapped_exception: Final = self._map_error_event_exception(error_obj)
         if not _mid_stream_fallback_eligible(mapped_exception):
             raise mapped_exception
+        mapped_status: Final = getattr(mapped_exception, "status_code", None)
         raise MidStreamFallbackError(
             message=str(mapped_exception),
             model=self.model or "",
             llm_provider=self.custom_llm_provider or "",
             original_exception=mapped_exception,
+            response=httpx.Response(
+                status_code=int(mapped_status) if mapped_status is not None else 503,
+                headers=provider_request_id_headers(self._raw_response_headers),
+                request=httpx.Request(method="POST", url=f"https://{self.custom_llm_provider or ''}.com/v1/"),
+            ),
             generated_content=self._generated_content,
             is_pre_first_chunk=not self._yielded_first_chunk,
         )

litellm/router.py

@@ -38,7 +38,7 @@ from dataclasses import dataclass
 from datetime import datetime, timezone
 from functools import lru_cache, partial
 from types import MappingProxyType
-from typing import TYPE_CHECKING, Any, Final, Generic, Literal, Optional, TypeAlias, TypeVar, Union, cast
+from typing import TYPE_CHECKING, Any, Final, Generic, Literal, NoReturn, Optional, TypeAlias, TypeVar, Union, cast
 
 import anyio
 import httpx
@@ -68,6 +68,7 @@ from litellm.constants import (
     DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIER,
     DEFAULT_MAX_LRU_CACHE_SIZE,
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
+    MID_STREAM_FAILURE_LOG_WAIT_SECONDS,
     OUTPUT_TOKEN_CEILING_PARAMS,
     PROMPT_CACHE_PIN_CACHE_MAX_SIZE,
     ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY,
@@ -75,7 +76,7 @@ from litellm.constants import (
     RUNTIME_UPDATABLE_ROUTER_SETTINGS,
     SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY,
 )
-from litellm.integrations.custom_guardrail import is_guardrail_intervention
+from litellm.integrations.custom_guardrail import CustomGuardrail, is_guardrail_intervention
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.integrations.otel.routing import routing_decision_attributes
 from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span
@@ -128,6 +129,7 @@ from litellm.llms.base_llm.vector_store.transformation import (
     RouterVectorStoreEmbeddingExecutor,
     vector_store_request_metadata,
 )
+from litellm.llms.bedrock.common_utils import get_bedrock_base_model
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
 from litellm.llms.openai_like.json_loader import JSONProviderRegistry
 from litellm.llms.openai_like.model_info import (
@@ -193,6 +195,7 @@ from litellm.router_utils.clientside_credential_handler import (
     is_clientside_credential,
 )
 from litellm.router_utils.common_utils import (
+    MID_STREAM_CONTINUATION_KWARG,
     filter_team_based_models,
     filter_web_search_deployments,
     format_fallback_outcome_message,
@@ -232,7 +235,10 @@ from litellm.router_utils.fallback_event_handlers import (
     check_non_standard_fallback_format,
     clear_pre_routing_selection,
     committed_retry_budget_for_request,
+    continuation_fallback_targets,
     fallback_lookup_groups,
+    fallback_target_name,
+    fallback_target_overrides_request_params,
     fallbacks_disabled_for_request,
     get_fallback_model_group_for_lookup_groups,
     get_pre_routing_selection,
@@ -257,6 +263,17 @@ from litellm.router_utils.handle_error import (
     send_llm_exception_alert,
 )
 from litellm.router_utils.health_state_cache import DeploymentHealthCache
+from litellm.router_utils.pre_call_checks.continuation_prefill_check import (
+    MID_STREAM_CONTINUATION_MARKER,
+    ContinuationOutput,
+    ContinuationPrefillDeploymentCheck,
+    ContinuedPartial,
+    constrains_continuation,
+    continuation_deployments,
+    continuation_output_ceilings,
+    continued_text,
+    deployment_can_continue,
+)
 from litellm.router_utils.pre_call_checks.deployment_affinity_check import (
     DeploymentAffinityCheck,
     warn_on_unknown_model_group_affinity_flags,
@@ -302,6 +319,8 @@ from litellm.router_utils.routing_groups import (
 )
 from litellm.router_utils.routing_read_batch import ROUTER_USAGE_TARGET, RoutingPrefetch, RoutingReadBatch
 from litellm.scheduler import FlowItem, Scheduler
+from litellm.secret_managers.main import str_to_bool
+from litellm.types.guardrails import GuardrailEventHooks
 from litellm.types.litellm_params import RoutingStrategyName
 from litellm.types.llms.openai import (
     AllMessageValues,
@@ -328,6 +347,7 @@ from litellm.types.router import (
     FallbackBudgetCheck,
     GuardrailTypedDict,
     LiteLLM_Params,
+    LiteLLMParamsTypedDict,
     MockRouterTestingParams,
     ModelGroupInfo,
     OptionalPreCallChecks,
@@ -369,11 +389,13 @@ from litellm.utils import (
     EmbeddingResponse,
     ModelResponse,
     Rules,
+    TextCompletionResponse,
     function_setup,
     get_llm_provider,
     get_non_default_completion_params,
     get_secret,
     get_utc_datetime,
+    is_generalized_model_info,
     is_region_allowed,
     provider_rejectable_params,
     set_live_deployment_replay,
@@ -484,6 +506,7 @@ _CLAUDE_CODE_SESSION_ID_RE: Final = re.compile(r"^[a-zA-Z0-9_\-]{8,}$")
 _CLAUDE_CODE_SESSION_ROUTER_TTL_SECONDS: Final = 3600
 CLAUDE_CODE_SESSION_ROUTER_BINDING_TARGET: Final = "claude_code_session_router_binding"
 
+_PRE_CALL_CHECK_FILTERED_PARAMS: Final = frozenset({"response_format"})
 _RUNTIME_TOGGLEABLE_PRE_CALL_CHECKS: Final[Mapping[str, type[CustomLogger]]] = MappingProxyType(
     {
         "prompt_caching": PromptCachingDeploymentCheck,
@@ -523,6 +546,15 @@ _ROUTING_KWARGS_ADAPTER: Final[TypeAdapter[Mapping[str, object] | None]] = TypeA
 _FALLBACK_HOP_ADAPTER: Final = TypeAdapter(Mapping[str, object])
 _OPAQUE_VALUE_ADAPTER: Final = TypeAdapter(object)
 _DEPLOYMENT_SELECTED_EVENT: Final = "litellm.request.deployment_selected"
+_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
+
+
+def _extend_trailing_text_block(blocks: Sequence[Mapping[str, object]], text: str) -> Sequence[Mapping[str, object]]:
+    """Continue the last text block in place so the prefill reads as the text the caller already received."""
+    last: Final = blocks[-1] if blocks else None
+    if last is not None and last.get("type") == "text" and isinstance(last_text := last.get("text"), str):
+        return [*blocks[:-1], {**last, "text": f"{last_text}{text}"}]
+    return [*blocks, {"type": "text", "text": text}]
 
 
 def _is_fallback_hop(request_kwargs: Mapping[str, object]) -> bool:
@@ -991,6 +1023,49 @@ def as_output_cap(value: object) -> int | None:
     return cap if cap >= 0 else None
 
 
+def _exact_base_model_max_input_tokens(base_model: str) -> int | None:
+    try:
+        model_info: Final = litellm.get_model_info(model=base_model)
+    except Exception:  # noqa: BLE001  # get_model_info raises ModelNotMappedError or a bare Exception for an unmapped model
+        return None
+    if is_generalized_model_info(model_info):
+        return None
+    return model_info.get("max_input_tokens")
+
+
+def _provider_model_id(model_name: str, custom_llm_provider: str) -> str:
+    model_id: Final = model_name.rsplit("/", 1)[-1]
+    return get_bedrock_base_model(model_id) if custom_llm_provider.startswith("bedrock") else model_id
+
+
+def _names_route_model(base_model: str, route_key: str, custom_llm_provider: str) -> bool:
+    route_model_id: Final = _provider_model_id(route_key, custom_llm_provider)
+    base_model_id: Final = _provider_model_id(base_model, custom_llm_provider)
+    return route_model_id in (base_model_id, base_model_id.split("@", 1)[0])
+
+
+def _declared_base_model(user_model_info: Mapping[str, object]) -> str | None:
+    match user_model_info:
+        case {"base_model": str() as base_model}:
+            return base_model
+        case _:
+            return None
+
+
+def _base_model_widened_window(
+    route_model_info: ModelMapInfo, base_model: str | None, custom_llm_provider: str
+) -> Mapping[str, int]:
+    route_max_input_tokens: Final = route_model_info.get("max_input_tokens")
+    if base_model is None or route_max_input_tokens is None:
+        return MappingProxyType({})
+    if not _names_route_model(base_model, route_model_info["key"], custom_llm_provider):
+        return MappingProxyType({})
+    base_max_input_tokens: Final = _exact_base_model_max_input_tokens(base_model)
+    if base_max_input_tokens is None or base_max_input_tokens <= route_max_input_tokens:
+        return MappingProxyType({})
+    return MappingProxyType({"max_input_tokens": base_max_input_tokens})
+
+
 class Router:
     @property
     def _routing_groups(self) -> dict[str, RoutingGroup]:
@@ -1072,6 +1147,7 @@ class Router:
         health_check_ignore_transient_errors: bool = False,
         background_health_check_model_groups: Sequence[str] | None = None,
         enable_weighted_failover: bool = False,
+        enable_mid_stream_fallback_continuation: bool = False,
         fallback_access_check: FallbackAccessCheck | None = None,
         fallback_budget_check: FallbackBudgetCheck | None = None,
         auto_router_capability_limit: AutoRouterCapabilityLimit | None = None,
@@ -1113,6 +1189,7 @@ class Router:
             deployment_affinity_ttl_seconds (int): TTL for user-key -> deployment affinity mapping. Defaults to 3600.
             ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error.
             enable_weighted_failover (bool): When True and the routing strategy is "simple-shuffle", a retryable failure on one deployment causes the request to re-pick (weighted) across the other deployments in the same model group before any cross-group fallback runs. Bounded by `max_fallbacks`. Async-only: currently honored by `router.acompletion()` and other async entrypoints. The sync `router.completion()` path falls back to the regular fallback flow. Defaults to False.
+            enable_mid_stream_fallback_continuation (bool): When True, a chat-completions stream that breaks after plain assistant text has been delivered continues on a fallback deployment via assistant prefill instead of surfacing the error. Only deployments whose model supports assistant prefill are eligible, so the partial text is continued, not regenerated; if none is, the original error is surfaced. Streams that emitted tool calls, thinking blocks, audio/images, or constrained (JSON / forced tool_choice) output are never continued. Async-only. Defaults to False.
             fallback_access_check (Optional[FallbackAccessCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects is skipped. Defaults to None (every configured fallback is attempted).
             fallback_budget_check (Optional[FallbackBudgetCheck]): Awaited before each cross-model-group fallback attempt on the async path; a fallback target it rejects as over budget is skipped. Defaults to None (budget is not re-checked on fallback).
         Returns:
@@ -1308,6 +1385,7 @@ class Router:
         self.fallback_on_unsupported_params = fallback_on_unsupported_params
         self.enable_health_check_routing = enable_health_check_routing
         self.enable_weighted_failover = enable_weighted_failover
+        self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enable_mid_stream_fallback_continuation)
         self.health_check_ignore_transient_errors = health_check_ignore_transient_errors
         self.background_health_check_model_groups: frozenset[str] | None = (
             frozenset(background_health_check_model_groups)
@@ -1502,6 +1580,14 @@ class Router:
         default_pre_call_checks: Final[OptionalPreCallChecks] = []
         self.add_optional_pre_call_checks(default_pre_call_checks)
 
+        if self.enable_mid_stream_fallback_continuation:
+            self._register_continuation_prefill_check()
+
+    @staticmethod
+    def _register_continuation_prefill_check() -> None:
+        # Process-global on purpose: Router.discard() must not drop a filter another router still needs.
+        litellm.logging_callback_manager.add_litellm_callback(ContinuationPrefillDeploymentCheck())
+
     def discard(self):
         """
         Pseudo-destructor to be invoked to clean up global data structures when router is no longer used.
@@ -3096,15 +3182,53 @@ class Router:
 
         usage: Final = cast(Usage | None, getattr(fallback_item, "usage", None))
         usage_objects: Final = [usage] if usage is not None else []
-        if (
-            complete_response_object_usage is not None
-            and hasattr(complete_response_object_usage, "usage")
-            and complete_response_object_usage.usage is not None
-        ):
+        if complete_response_object_usage is not None:
             usage_objects.append(complete_response_object_usage)
         combined_usage: Final = BaseTokenUsageProcessor.combine_usage_objects(usage_objects=usage_objects)
         setattr(fallback_item, "usage", combined_usage)
 
+    @staticmethod
+    def _price_cut_partial(
+        cut_stream: CustomStreamWrapper,
+        partial_response: ModelResponse | TextCompletionResponse | None,
+    ) -> "ContinuedPartial | None":
+        """Price the cut stream's partial, plus any partial it inherited, before a fallback can reuse its logging."""
+        from litellm.cost_calculator import BaseTokenUsageProcessor
+
+        cut_logging: Final = cut_stream.logging_obj
+        partial_usage: Final = getattr(partial_response, "usage", None)
+        if partial_response is None or not isinstance(partial_usage, Usage):
+            return None
+        inherited_usage: Final = cut_logging.model_call_details.get("continued_partial_usage")
+        inherited_cost: Final = cut_logging.model_call_details.get("continued_partial_cost")
+        inherited_id: Final = cut_logging.model_call_details.get("continued_response_id")
+        priced_partial: Final = partial_response.model_copy(
+            update={"model": cut_stream.model} if cut_stream.model else {}
+        )
+        priced_partial._hidden_params = {  # pyright: ignore[reportPrivateUsage]  # no public accessor
+            key: value for key, value in get_hidden_params_dict(partial_response).items() if key != "response_cost"
+        }
+        return ContinuedPartial(
+            usage=BaseTokenUsageProcessor.combine_usage_objects(
+                usage_objects=[partial_usage, inherited_usage]
+                if isinstance(inherited_usage, Usage)
+                else [partial_usage]
+            ),
+            response_cost=(cut_logging.response_cost_calculator(result=priced_partial) or 0.0)
+            + (inherited_cost if isinstance(inherited_cost, float) else 0.0),
+            response_id=inherited_id if isinstance(inherited_id, str) else partial_response.id,
+        )
+
+    @staticmethod
+    def _carry_partial_spend_to_fallback(fallback_response: object, cut_partial: "ContinuedPartial | None") -> None:
+        """Bill what the cut legs consumed on the continuation's success log."""
+        fallback_logging: Final = getattr(fallback_response, "logging_obj", None)
+        if cut_partial is None or not isinstance(fallback_logging, LiteLLMLogging):
+            return
+        fallback_logging.record_continued_partial(
+            usage=cut_partial.usage, response_cost=cut_partial.response_cost, response_id=cut_partial.response_id
+        )
+
     @staticmethod
     def _prepare_fallback_hidden_params(
         fallback_response: object,
@@ -3154,7 +3278,7 @@ class Router:
         self,
         model_response: CustomStreamWrapper,
         messages: list[dict[str, str]],
-        initial_kwargs: dict,
+        initial_kwargs: dict[str, Any],
         deployment_slot: contextlib.AsyncExitStack | None = None,
         first_chunk: _FirstChunk[AsyncIterator[object]] | None = None,
     ) -> CustomStreamWrapper:
@@ -3217,16 +3341,22 @@ class Router:
                 with anyio.CancelScope(shield=True):
                     await close_model_response()
                     await held_slot.aclose()
-                if not e.is_pre_first_chunk and (
-                    e.generated_content or _stream_chunks_have_generated_content(model_response.chunks)
-                ):
-                    if e.original_exception is not None:
-                        raise e.original_exception from e
-                    raise
+                committed: Final = bool(
+                    not e.is_pre_first_chunk
+                    and (e.generated_content or _stream_chunks_have_generated_content(model_response.chunks))
+                )
+                cut_hidden_params: Final = get_hidden_params(model_response) or {}
+                continue_after_content: Final = committed and self._mid_stream_continuation_eligible(
+                    e=e, request_kwargs=initial_kwargs, cut_deployment_id=cut_hidden_params.get("model_id")
+                )
+                if committed and not continue_after_content:
+                    self._raise_original_mid_stream_error(e)
 
                 from litellm.main import stream_chunk_builder
 
-                complete_response_object: Final = stream_chunk_builder(chunks=model_response.chunks)
+                complete_response_object: Final = stream_chunk_builder(
+                    chunks=model_response.chunks, messages=messages if continue_after_content else None
+                )
                 complete_response_object_usage: Final = cast(
                     Usage | None,
                     getattr(complete_response_object, "usage", None),
@@ -3242,7 +3372,41 @@ class Router:
                         "content_policy_fallbacks", self.content_policy_fallbacks
                     )
                     initial_kwargs["original_function"] = self._acompletion
-                    initial_kwargs["messages"] = messages
+                    continuation_messages: Final = (
+                        self._build_completion_continuation_input(messages, e.generated_content)
+                        if continue_after_content
+                        else None
+                    )
+                    continuation_output: Final = (
+                        ContinuationOutput(
+                            e.generated_content,
+                            continuation_messages[-1],
+                            model_group,
+                            complete_response_object,
+                            checks_post_call_rules=initial_kwargs.get(MID_STREAM_CONTINUATION_KWARG)
+                            is not MID_STREAM_CONTINUATION_MARKER,
+                        )
+                        if continuation_messages is not None
+                        else None
+                    )
+                    cut_partial: Final = (
+                        Router._price_cut_partial(model_response, complete_response_object)
+                        if continuation_output is not None
+                        else None
+                    )
+                    if continue_after_content:
+                        emitted_tokens: Final = int(
+                            getattr(complete_response_object_usage, "completion_tokens", 0) or 0
+                        )
+                        reduced_ceilings: Final = continuation_output_ceilings(initial_kwargs, emitted_tokens)
+                        if reduced_ceilings is None or continuation_messages is None:
+                            self._raise_original_mid_stream_error(e)
+                        await model_response.wait_for_failure_logging(timeout=MID_STREAM_FAILURE_LOG_WAIT_SECONDS)
+                        initial_kwargs.update(reduced_ceilings)
+                        initial_kwargs["messages"] = continuation_messages
+                        initial_kwargs[MID_STREAM_CONTINUATION_KWARG] = MID_STREAM_CONTINUATION_MARKER
+                    else:
+                        initial_kwargs["messages"] = messages
                     self._update_kwargs_before_fallbacks(model=model_group, kwargs=initial_kwargs)
                     fallback_response = await self.async_function_with_fallbacks_common_utils(
                         e=e,
@@ -3258,11 +3422,17 @@ class Router:
 
                     # If fallback returns a streaming response, iterate over it
                     if hasattr(fallback_response, "__aiter__"):
+                        Router._carry_partial_spend_to_fallback(fallback_response, cut_partial)
                         prepared_fallback_hidden_params = Router._adopt_fallback_response_headers(
                             wrapper_ref, fallback_response
                         )
                         fallback_headers_are_settled = False
-                        async for fallback_item in fallback_response:
+                        fallback_items: Final = (
+                            continuation_output.splice(fallback_response)
+                            if continuation_output is not None
+                            else fallback_response
+                        )
+                        async for fallback_item in fallback_items:
                             if not fallback_headers_are_settled:
                                 fallback_headers_are_settled = True
                                 # a fallback that failed over again only repoints itself once it yields
@@ -3275,7 +3445,10 @@ class Router:
                                 and isinstance(fallback_item, ModelResponseStream)
                                 and hasattr(fallback_item, "usage")
                             ):
-                                self._combine_fallback_usage(fallback_item, complete_response_object_usage)
+                                self._combine_fallback_usage(
+                                    fallback_item,
+                                    complete_response_object_usage if continuation_output is not None else None,
+                                )
                             yield fallback_item
                     else:
                         # If fallback returns a non-streaming response, yield None
@@ -3432,6 +3605,97 @@ class Router:
             total_tokens=(partial_usage.total_tokens or 0) + (fb.total_tokens or 0),
         )
 
+    @staticmethod
+    def _raise_original_mid_stream_error(e: "MidStreamFallbackError") -> NoReturn:
+        """Surface the provider error the stream wrapper carried instead of the internal MidStreamFallbackError."""
+        if e.original_exception is not None:
+            raise e.original_exception from e
+        raise e
+
+    def _params_sent_to_deployment(
+        self, request_kwargs: Mapping[str, object], deployment: Deployment | None
+    ) -> Mapping[str, object]:
+        """The request as the deployment received it, merged in the order _update_kwargs_with_deployment applies."""
+        deployment_params: Final = deployment.litellm_params.model_dump(exclude_none=True) if deployment else {}
+        defaults: Final = {key: value for key, value in self.default_litellm_params.items() if value is not None}
+        deployment_tool_choice: Final = (
+            {"tool_choice": deployment_params["tool_choice"]}
+            if deployment_params.get("tool_choice") is not None
+            else {}
+        )
+        return MappingProxyType({**deployment_params, **defaults, **deployment_tool_choice, **request_kwargs})
+
+    def _names_deployment_that_cannot_continue(self, target: object) -> bool:
+        name: Final = fallback_target_name(target)
+        if name is None or name in self.model_names or not self.has_model_id(name):
+            return False
+        deployment: Final = self.get_deployment(model_id=name)
+        return deployment is not None and not deployment_can_continue(deployment.model_dump(exclude_none=True))
+
+    def _mid_stream_continuation_eligible(
+        self,
+        e: "MidStreamFallbackError",
+        request_kwargs: Mapping[str, object],
+        cut_deployment_id: object = None,
+    ) -> bool:
+        """Whether a stream that broke after plain assistant text may continue via prefill."""
+        if not self.enable_mid_stream_fallback_continuation:
+            return False
+        if not e.generated_content or e.emitted_disqualifying_content:
+            return False
+        if isinstance(e.original_exception, litellm.ContentPolicyViolationError):
+            return False
+        cut_deployment: Final = (
+            self.get_deployment(model_id=cut_deployment_id) if isinstance(cut_deployment_id, str) else None
+        )
+        if isinstance(cut_deployment_id, str) and cut_deployment is None:
+            return False
+        if constrains_continuation(self._params_sent_to_deployment(request_kwargs, cut_deployment)):
+            return False
+        requested_group: Final = request_kwargs.get("model")
+        lookup_groups: Final = fallback_lookup_groups(
+            request_kwargs, requested_group if isinstance(requested_group, str) else None
+        )
+        targets: Final = continuation_fallback_targets(request_kwargs.get("fallbacks", self.fallbacks), lookup_groups)
+        if any(fallback_target_overrides_request_params(target) for target in targets):
+            return False
+        if any(self._names_deployment_that_cannot_continue(target) for target in targets):
+            return False
+        return not any(
+            isinstance(callback, CustomGuardrail)
+            and callback._event_hook_is_event_type(  # pyright: ignore[reportPrivateUsage]  # the only check of a guardrail's hooks
+                GuardrailEventHooks.post_call
+            )
+            for callback in litellm.callbacks
+        )
+
+    @staticmethod
+    def _build_completion_continuation_input(
+        messages: Sequence[Mapping[str, object]],
+        generated_content: str,
+    ) -> Sequence[Mapping[str, object]] | None:
+        """Append the partial output as an assistant prefill, or extend a trailing `prefix: True` assistant turn in
+        place so the request never ends in two assistant messages. None when the request ends in a plain assistant
+        turn, in a prefill turn whose content has a shape this cannot extend, or when the cut came mid-echo of the
+        prefill."""
+        last: Final = messages[-1] if messages else None
+        if last is None or last.get("role") != "assistant":
+            return [*messages, {"role": "assistant", "content": generated_content, "prefix": True}]
+        if last.get("prefix") is not True:
+            return None
+        new_text: Final = continued_text(last, generated_content)
+        if new_text is None:
+            return None
+        content: Final = last.get("content")
+        if content is None or isinstance(content, str):
+            return [*messages[:-1], {**last, "content": f"{content or ''}{new_text}", "prefix": True}]
+        try:
+            blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(content)
+        except ValidationError:
+            return None
+        extended: Final = _extend_trailing_text_block(blocks, new_text)
+        return [*messages[:-1], {**last, "content": extended, "prefix": True}]
+
     @staticmethod
     def _build_responses_continuation_input(
         input_val: Union[str, "ResponseInputParam"] | None,
@@ -3833,13 +4097,6 @@ class Router:
                         raise e.original_exception from e
                     raise
 
-                from litellm.main import stream_chunk_builder
-
-                complete_response_object: Final = stream_chunk_builder(chunks=model_response.chunks)
-                complete_response_object_usage: Final = cast(
-                    Usage | None,
-                    getattr(complete_response_object, "usage", None),
-                )
                 try:
                     model_group: Final = cast(str, initial_kwargs.get("model"))
                     fallbacks: Final[list | None] = initial_kwargs.get("fallbacks", router_self.fallbacks)
@@ -3885,7 +4142,7 @@ class Router:
                                 and isinstance(fallback_item, ModelResponseStream)
                                 and hasattr(fallback_item, "usage")
                             ):
-                                router_self._combine_fallback_usage(fallback_item, complete_response_object_usage)
+                                router_self._combine_fallback_usage(fallback_item, None)
                             yield fallback_item
                     else:
                         yield None
@@ -11343,6 +11600,7 @@ class Router:
         Augment litellm info with additional params set in `model_info`.
 
         For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set.
+        For every other provider, a base_model naming the same model as `model:` can only widen its context window.
 
         Returns
         - ModelInfo - If found -> typed dict with max tokens, input cost, etc.
@@ -11439,20 +11697,23 @@ class Router:
 
         ## CHECK USER SET MODEL INFO
         raw_user_model_info: Final = deployment.get("model_info")
-        user_model_info: Final = (
+        user_model_info: Final = _MODEL_INFO_ADAPTER.validate_python(
             raw_user_model_info.model_dump(exclude_none=True)
             if isinstance(raw_user_model_info, BaseModel)
-            else raw_user_model_info
+            else raw_user_model_info or {}
         )
 
         # get_model_info() hands back an lru_cache'd dict, so merge into a copy; unset
         # values are skipped or Deployment's None pricing defaults would erase the map's
         merged_model_info: Final[ModelMapInfo] = {
             **copy.deepcopy(model_info),
-            **self.get_discovered_model_info((deployment.get("model_info") or {}).get("id")),
-            **MappingProxyType(
-                {key: value for key, value in (user_model_info or MappingProxyType({})).items() if value is not None}
+            **_base_model_widened_window(
+                model_info,
+                _declared_base_model(user_model_info) or _declared_base_model(litellm_params.model_extra or {}),
+                custom_llm_provider,
             ),
+            **self.get_discovered_model_info((deployment.get("model_info") or {}).get("id")),
+            **MappingProxyType({key: value for key, value in user_model_info.items() if value is not None}),
         }
 
         return merged_model_info
@@ -12642,16 +12903,34 @@ class Router:
             str(deployment_params.get("model") or ""), deployment_params.get("custom_llm_provider")
         ):
             return True
+        return Router._deployment_supports_param(deployment, group, param)
+
+    @staticmethod
+    def _capability_model_and_provider(deployment_params: LiteLLMParamsTypedDict, group: str) -> tuple[str, str]:
+        model_label: Final = str(deployment_params.get("model") or group)
+        authenticating_provider: Final = declared_authenticating_provider(
+            model_label, deployment_params.get("custom_llm_provider")
+        )
+        if authenticating_provider is not None:
+            return model_label.removeprefix(f"{authenticating_provider}/"), authenticating_provider
+        model, custom_llm_provider, _, _ = litellm.get_llm_provider(
+            model=model_label,
+            custom_llm_provider=deployment_params.get("custom_llm_provider"),
+            litellm_params=LiteLLM_Params.model_validate(deployment_params),
+        )
+        return model, custom_llm_provider
+
+    @staticmethod
+    def _deployment_supports_param(deployment: DeploymentTypedDict, group: str, param: str) -> bool:
+        deployment_params: Final = deployment.get("litellm_params")
+        if not deployment_params:
+            return True
         deployment_model_info: Final = deployment.get("model_info")
         base_model: Final = (
             deployment_model_info.get("base_model") if deployment_model_info else None
         ) or deployment_params.get("base_model")
         try:
-            model, custom_llm_provider, _, _ = litellm.get_llm_provider(
-                model=deployment_params.get("model") or group,
-                custom_llm_provider=deployment_params.get("custom_llm_provider"),
-                litellm_params=LiteLLM_Params.model_validate(deployment_params),
-            )
+            model, custom_llm_provider = Router._capability_model_and_provider(deployment_params, group)
             supported: Final = litellm.get_supported_openai_params(
                 model=model,
                 custom_llm_provider=custom_llm_provider,
@@ -12659,7 +12938,7 @@ class Router:
             )
         except Exception as e:  # noqa: BLE001  # best-effort filter: an unresolvable provider must not narrow the request
             verbose_router_logger.debug(
-                "litellm.router.py::_deployment_accepts_param: keeping %s for model=%s. Got - %s", param, group, e
+                "litellm.router.py::_deployment_supports_param: keeping %s for model=%s. Got - %s", param, group, e
             )
             return True
         return supported is None or param in supported
@@ -12935,6 +13214,7 @@ class Router:
             "retry_policy",
             "model_group_alias",
             "enable_weighted_failover",
+            "enable_mid_stream_fallback_continuation",
             "enable_tag_filtering",
             "tag_routing_prefix",
             "fallback_on_unsupported_params",
@@ -12991,12 +13271,10 @@ class Router:
                     rebuild_routing_groups = True
                 elif var == "optional_pre_call_checks":
                     self.set_optional_pre_call_checks(kwargs[var])
+                elif var == "enable_mid_stream_fallback_continuation":
+                    self._set_mid_stream_fallback_continuation(kwargs[var])
                 elif var == "retry_policy":
-                    value = kwargs[var]
-                    if isinstance(value, dict):
-                        value = RetryPolicy(**value)
-                    if value is None or isinstance(value, RetryPolicy):
-                        setattr(self, var, value)
+                    self._set_retry_policy(kwargs[var])
                 else:
                     value = kwargs[var]
                     # only run routing strategy init if it has changed
@@ -13023,6 +13301,20 @@ class Router:
             self._routing_groups_input = routing_groups_input
         verbose_router_logger.debug("Updated Router settings: %s", self.get_settings())
 
+    @staticmethod
+    def _is_flag_enabled(value: object) -> bool:
+        return value is True or (isinstance(value, str) and str_to_bool(value) is True)
+
+    def _set_mid_stream_fallback_continuation(self, enabled: object) -> None:
+        self.enable_mid_stream_fallback_continuation = self._is_flag_enabled(enabled)
+        if self.enable_mid_stream_fallback_continuation:
+            self._register_continuation_prefill_check()
+
+    def _set_retry_policy(self, value: object) -> None:
+        policy: Final = RetryPolicy.model_validate(value) if isinstance(value, dict) else value
+        if policy is None or isinstance(policy, RetryPolicy):
+            self.retry_policy = policy
+
     def _get_client(self, deployment, kwargs, client_type=None):
         """
         Returns the appropriate client based on the given deployment, kwargs, and client_type.
@@ -13156,7 +13448,8 @@ class Router:
 
         Returns None when no deployment limits its context window, and when counting fails. The
         caller pairs this with ``skip_inline_token_count`` so neither case puts the count back on
-        the loop: a failed count leaves the deployments unfiltered, exactly as before.
+        the loop: a failed count only skips the context-window check - RPM, region and
+        invalid-params checks still run for every deployment.
         """
         if messages is None and input is None:
             return None
@@ -13174,6 +13467,66 @@ class Router:
             )
             return None
 
+    def _resolve_pre_call_check_input_tokens(
+        self,
+        messages: Sequence[Mapping[str, str]] | None,
+        input: str | Sequence[object] | None,
+        request_kwargs: Mapping[str, object] | None,
+        input_token_count: int | None,
+        count_inline: bool,
+    ) -> int | None:
+        if input_token_count is not None:
+            return input_token_count
+        if not count_inline:
+            return None
+        try:
+            return self._count_pre_call_check_tokens(
+                messages=cast(list[dict[str, str]] | None, messages),  # cast-ok: forwarded to the sync counter
+                input=cast(str | list[object] | None, input),  # cast-ok: forwarded to the sync counter
+                request_kwargs=request_kwargs,
+            )
+        except Exception as e:
+            verbose_router_logger.error(
+                "litellm.router.py::_pre_call_checks: failed to count tokens. "
+                "Skipping the context-window check only; RPM/region/param checks still apply. Got - %s",
+                e,
+            )
+            return None
+
+    @staticmethod
+    def _params_neither_dropped_nor_allowed(params: Mapping[str, object], candidates: frozenset[str]) -> frozenset[str]:
+        if params.get("drop_params") is True:
+            return frozenset()
+        dropped: Final = params.get("additional_drop_params")
+        dropped_names: Final = (
+            frozenset(entry for entry in dropped if isinstance(entry, str))
+            if isinstance(dropped, list)
+            else frozenset()
+        )
+        return candidates - Router._declared_param_allowlist(params) - dropped_names
+
+    @staticmethod
+    def _deployment_lacks_params(deployment: DeploymentTypedDict, group: str, params: frozenset[str]) -> bool:
+        return any(not Router._deployment_supports_param(deployment, group, param) for param in params)
+
+    @staticmethod
+    def _deployment_rejects_params(
+        deployment: DeploymentTypedDict, group: str, params: frozenset[str], request_kwargs: Mapping[str, object]
+    ) -> bool:
+        call_settings: Final[Mapping[str, object]] = {**(deployment.get("litellm_params") or {}), **request_kwargs}
+        return Router._deployment_lacks_params(
+            deployment, group, Router._params_neither_dropped_nor_allowed(call_settings, params)
+        )
+
+    @staticmethod
+    def _pre_call_check_requested_chat_params(request_kwargs: Mapping[str, object]) -> frozenset[str]:
+        requested: Final = litellm.utils.get_non_default_params(passed_params=dict(request_kwargs))
+        return frozenset(
+            param
+            for param, value in requested.items()
+            if param in _PRE_CALL_CHECK_FILTERED_PARAMS and not isinstance(value, str)
+        )
+
     def _pre_call_checks(
         self,
         model: str,
@@ -13183,6 +13536,7 @@ class Router:
         request_kwargs: dict | None = None,
         input_token_count: int | None = None,
         skip_inline_token_count: bool = False,
+        request_deployments: Sequence[DeploymentTypedDict] | None = None,
     ):
         """
         Filter out model in model group, if:
@@ -13201,13 +13555,6 @@ class Router:
 
         invalid_model_indices: Final = set()  # Use set for O(1) membership checks
 
-        # Token counting (tiktoken) is the dominant on-loop cost for large prompts.
-        # Only count when a deployment actually declares max_input_tokens, and count
-        # at most once; for model groups with no context-window limit it is skipped.
-        # Async callers pass the count in, already computed off the event loop, and set
-        # skip_inline_token_count so a failed off-loop count is not retried back on the loop.
-        input_tokens: int | None = input_token_count
-
         _context_window_error = False
         _potential_error_str = ""
         _rate_limit_error = False
@@ -13216,6 +13563,24 @@ class Router:
         has_countable_input: Final = (messages is not None or input is not None) and not compaction_pending(
             request_kwargs
         )
+        deployment_max_input_tokens: Final = tuple(
+            self._deployment_max_input_tokens(model, deployment) for deployment in _returned_deployments
+        )
+        input_tokens: Final = self._resolve_pre_call_check_input_tokens(
+            messages=messages,
+            input=cast(str | list[object] | None, input),  # cast-ok: forwarded to the resolver
+            request_kwargs=cast(Mapping[str, object] | None, request_kwargs),  # cast-ok: forwarded to the resolver
+            input_token_count=input_token_count,
+            count_inline=has_countable_input
+            and not skip_inline_token_count
+            and any(limit is not None for limit in deployment_max_input_tokens),
+        )
+        request_settings: Final[Mapping[str, object]] = request_kwargs if request_kwargs is not None else {}
+        requested_params: Final[frozenset[str]] = (
+            self._pre_call_check_requested_chat_params(request_settings)
+            if request_kwargs is not None and litellm.drop_params is False
+            else frozenset()
+        )
 
         ## get model group RPM ##
         dt: Final = get_utc_datetime()
@@ -13229,39 +13594,22 @@ class Router:
             _litellm_params = deployment.get("litellm_params", {})
             _model_info = deployment.get("model_info", {})
 
-            # see if we have the info for this model
-            _deployment_model = None  # per-deployment model name (avoids overwriting the outer `model` group name)
-            try:
-                base_model = _model_info.get("base_model", None)
-                if base_model is None:
-                    base_model = _litellm_params.get("base_model", None)
-                _deployment_model = base_model or _litellm_params.get("model", None)
-                model_info = self.get_router_model_info(deployment=deployment, received_model_name=model)
-
-                max_input_tokens = model_info.get("max_input_tokens") if isinstance(model_info, dict) else None
-                if isinstance(max_input_tokens, int) and has_countable_input:
-                    if input_tokens is None:
-                        if skip_inline_token_count:
-                            return _returned_deployments
-                        try:
-                            input_tokens = self._count_pre_call_check_tokens(
-                                messages=messages, input=input, request_kwargs=request_kwargs
-                            )
-                        except Exception as e:
-                            verbose_router_logger.error(
-                                "litellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - %s",
-                                e,
-                            )
-                            return _returned_deployments
-                    if input_tokens > max_input_tokens:
-                        invalid_model_indices.add(idx)
-                        _context_window_error = True
-                        _potential_error_str += (
-                            f"Model={_deployment_model}, Max Input Tokens={max_input_tokens}, Got={input_tokens}"
-                        )
-                        continue
-            except Exception as e:
-                verbose_router_logger.exception("An error occurs - %s", e)
+            max_input_tokens = deployment_max_input_tokens[idx]
+            if (
+                max_input_tokens is not None
+                and has_countable_input
+                and input_tokens is not None
+                and input_tokens > max_input_tokens
+            ):
+                invalid_model_indices.add(idx)
+                _context_window_error = True
+                deployment_model = (
+                    _model_info.get("base_model") or _litellm_params.get("base_model") or _litellm_params.get("model")
+                )
+                _potential_error_str += (
+                    f"Model={deployment_model}, Max Input Tokens={max_input_tokens}, Got={input_tokens}"
+                )
+                continue
 
             model_id = _model_info.get("id", "")
             ## RPM CHECK ##
@@ -13293,53 +13641,57 @@ class Router:
                         invalid_model_indices.add(idx)
                         continue
 
-            ## INVALID PARAMS ## -> catch 'gpt-3.5-turbo-16k' not supporting 'response_format' param
-            if request_kwargs is not None and litellm.drop_params is False:
-                # get supported params — use per-deployment model to avoid overwriting the outer model group name
-                _dep_model_for_params: str = _deployment_model or model
-                try:
-                    (
-                        _dep_model_for_params,
-                        custom_llm_provider,
-                        _,
-                        _,
-                    ) = litellm.get_llm_provider(
-                        model=_dep_model_for_params,
-                        litellm_params=LiteLLM_Params.model_validate(_litellm_params),
-                    )
-                except Exception as e:  # noqa: BLE001  # best-effort filter: an unresolvable provider must not fail the request
-                    verbose_router_logger.debug(
-                        "litellm.router.py::_pre_call_checks: skipping supported-params check for model=%s. Got - %s",
-                        _dep_model_for_params,
-                        e,
-                    )
-                    continue
+        typed_deployments: Final = tuple(
+            cast(DeploymentTypedDict, deployment)  # cast-ok: router deployments are plain dicts
+            for deployment in _returned_deployments
+        )
+        param_candidates: Final = tuple(
+            (idx, deployment) for idx, deployment in enumerate(typed_deployments) if idx not in invalid_model_indices
+        )
+        candidates_lacking_params: Final = tuple(
+            self._deployment_lacks_params(deployment, model, requested_params) for _, deployment in param_candidates
+        )
+        no_deployment_takes_params: Final = bool(candidates_lacking_params) and all(candidates_lacking_params)
+        candidates_failing_params: Final = (
+            tuple(
+                self._deployment_rejects_params(deployment, model, requested_params, request_settings)
+                for _, deployment in param_candidates
+            )
+            if no_deployment_takes_params
+            else candidates_lacking_params
+        )
+        excluded_indices: Final = frozenset(invalid_model_indices) | frozenset(
+            idx for (idx, _), fails in zip(param_candidates, candidates_failing_params, strict=True) if fails
+        )
 
-                supported_openai_params = litellm.get_supported_openai_params(
-                    model=_dep_model_for_params,
-                    custom_llm_provider=custom_llm_provider,
-                )
-
-                if supported_openai_params is None:
-                    continue
-                else:
-                    # check the non-default openai params in request kwargs
-                    non_default_params = litellm.utils.get_non_default_params(passed_params=request_kwargs)
-                    special_params = ["response_format"]
-                    # check if all params are supported
-                    for k, v in non_default_params.items():
-                        if k not in supported_openai_params and k in special_params:
-                            # if not -> invalid model
-                            verbose_router_logger.debug("INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k=%s", k)
-                            invalid_model_indices.add(idx)
-
-        if len(invalid_model_indices) == len(_returned_deployments):
+        if len(excluded_indices) == len(_returned_deployments):
             """
             - no healthy deployments available b/c context window checks or rate limit error
 
             - First check for rate limit errors (if this is true, it means the model passed the context window check but failed the rate limit check)
             """
 
+            group_deployments: Final = (
+                tuple(request_deployments)
+                if request_deployments is not None
+                else (tuple(self.get_all_deployments(model_name=model)) or typed_deployments)
+            )
+            if (
+                not _context_window_error
+                and group_deployments
+                and all(
+                    self._deployment_rejects_params(deployment, model, requested_params, request_settings)
+                    for deployment in group_deployments
+                )
+            ):
+                raise litellm.UnsupportedParamsError(
+                    message=(
+                        f"no deployment of model={model} supports {sorted(requested_params)}. "
+                        "To drop these, set `litellm.drop_params=True` or for proxy `litellm_settings: drop_params: true`, "
+                        f"or send allowed_openai_params={sorted(requested_params)} to pass them through"
+                    ),
+                    model=model,
+                )
             if _rate_limit_error is True:  # allow generic fallback logic to take place
                 raise RouterRateLimitErrorBasic(
                     model=model,
@@ -13351,9 +13703,9 @@ class Router:
                     model=model,
                     llm_provider="",
                 )
-        if len(invalid_model_indices) > 0:
+        if len(excluded_indices) > 0:
             # Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops)
-            _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in invalid_model_indices]
+            _returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices]
 
         return _returned_deployments
 
@@ -13776,6 +14128,10 @@ class Router:
                 )
             return healthy_deployments
 
+        request_deployments: Final = cast(  # cast-ok: router deployments are plain dicts
+            Sequence[DeploymentTypedDict], healthy_deployments
+        )
+
         # Health-check-based filtering (before cooldown)
         healthy_deployments = await self._async_filter_health_check_unhealthy_deployments(
             healthy_deployments=healthy_deployments,
@@ -13835,6 +14191,7 @@ class Router:
                     request_kwargs=request_kwargs,
                 ),
                 skip_inline_token_count=True,
+                request_deployments=request_deployments,
             )
         # check if user wants to do tag based routing
         healthy_deployments = await get_deployments_for_tag(
@@ -14985,9 +15342,8 @@ class Router:
             request_kwargs=request_kwargs,
         )
         strategy, strategy_selector = self._get_routing_context(model, request_kwargs)
-        pick_attributes: Final = _deployment_pick_attributes(
-            model, _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs)
-        )
+        routing_kwargs: Final = _ROUTING_KWARGS_ADAPTER.validate_python(request_kwargs)
+        pick_attributes: Final = _deployment_pick_attributes(model, routing_kwargs)
 
         if isinstance(healthy_deployments, dict):
             if (healthy_deployments.get("model_info") or {}).get("blocked") is True:
@@ -15001,6 +15357,9 @@ class Router:
             return healthy_deployments
 
         parent_otel_span: Final[Span | None] = get_parent_otel_span_from_kwargs(request_kwargs)
+        request_deployments: Final = cast(  # cast-ok: router deployments are plain dicts
+            Sequence[DeploymentTypedDict], healthy_deployments
+        )
 
         # Health-check-based filtering (before cooldown)
         healthy_deployments = self._filter_health_check_unhealthy_deployments(
@@ -15022,7 +15381,9 @@ class Router:
             )
             healthy_deployments = _pre_cooldown_deployments
 
-        healthy_deployments = self._filter_blocked_deployments(healthy_deployments)
+        healthy_deployments = continuation_deployments(
+            self._filter_blocked_deployments(healthy_deployments), routing_kwargs
+        )
 
         # filter pre-call checks
         if self.enable_pre_call_checks and (messages is not None or input is not None):
@@ -15032,6 +15393,7 @@ class Router:
                 messages=messages,
                 input=input,
                 request_kwargs=request_kwargs,
+                request_deployments=request_deployments,
             )
 
         ## ORDER FILTERING ## -> if user set 'order' in deployments, return deployments with lowest order (e.g. order=1 > order=2)
@@ -15195,6 +15557,9 @@ class Router:
 
         # 4. Apply health-check and cooldown filtering
         parent_otel_span: Final[Span | None] = get_parent_otel_span_from_kwargs(request_kwargs)
+        request_deployments: Final = cast(  # cast-ok: router deployments are plain dicts
+            Sequence[DeploymentTypedDict], pass_through_deployments
+        )
         pass_through_deployments = self._filter_health_check_unhealthy_deployments(
             healthy_deployments=pass_through_deployments,
             parent_otel_span=parent_otel_span,
@@ -15216,6 +15581,7 @@ class Router:
                 messages=messages,
                 input=input,
                 request_kwargs=request_kwargs,
+                request_deployments=request_deployments,
             )
 
         if self._is_priority_routing_group(model):

litellm/router_strategy/budget_limiter.py

@@ -405,7 +405,8 @@ class RouterBudgetLimiting(CustomLogger):
         Runs once the budget start time exists in Redis Cache (on the 2nd and subsequent requests to the same provider)
 
         - Increments the spend in memory cache (so spend instantly updated in memory)
-        - Queues the increment operation to Redis Pipeline (using batched pipeline to optimize performance. Using Redis for multi instance environment of LiteLLM)
+        - Queues the increment for the batched Redis pipeline when Redis is configured, since only a multi instance
+          deployment has anything to sync
         """
         increment_op: Final = RedisPipelineIncrementOperation(
             key=spend_key,
@@ -418,6 +419,8 @@ class RouterBudgetLimiting(CustomLogger):
                 value=response_cost,
                 ttl=ttl,
             )
+            if self.dual_cache.redis_cache is None:
+                return
             self.redis_increment_operation_queue.append(increment_op)
 
     def _get_redis_increment_queue_lock(self) -> asyncio.Lock:

litellm/router_utils/common_utils.py

@@ -27,7 +27,8 @@ from litellm.types.utils import LlmProviders, ProviderSpecificModelInfo
 
 _V = TypeVar("_V")
 
-ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors"})
+MID_STREAM_CONTINUATION_KWARG: Final = "_mid_stream_continuation"
+ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors", MID_STREAM_CONTINUATION_KWARG})
 _OBJECT_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[object, object]]] = TypeAdapter(Mapping[object, object])
 
 

litellm/router_utils/fallback_event_handlers.py

@@ -615,6 +615,32 @@ def get_fallback_model_group(
     return fallback_model_group, generic_fallback_idx
 
 
+def _resolved_fallback_chain(entries: Sequence[object], lookup_groups: tuple[str, ...]) -> Sequence[object]:
+    group_entries: Final[Sequence[Mapping[str, list[str]]]] = tuple(  # mutable-ok: mirrors the fallbacks config shape
+        entry for entry in entries if isinstance(entry, Mapping) and entry
+    )
+    chain, _ = get_fallback_model_group_for_lookup_groups(fallbacks=group_entries, lookup_groups=lookup_groups)
+    return chain if isinstance(chain, list) else ()
+
+
+def continuation_fallback_targets(fallbacks: object, lookup_groups: tuple[str, ...]) -> Sequence[object]:
+    entries: Final[Sequence[object]] = fallbacks if isinstance(fallbacks, list) else ()
+    return (
+        entries
+        if check_non_standard_fallback_format(fallbacks=entries)
+        else _resolved_fallback_chain(entries, lookup_groups)
+    )
+
+
+def fallback_target_overrides_request_params(target: object) -> bool:
+    return isinstance(target, Mapping) and "model" in target and len(target) > 1
+
+
+def fallback_target_name(target: object) -> str | None:
+    name: Final = target.get("model") if isinstance(target, Mapping) else target
+    return name if isinstance(name, str) else None
+
+
 PROVIDER_SCOPED_RESOURCE_KEYS: Final = ("input_file_id", "training_file", "batch_id", "file_id", "fine_tuning_job_id")
 PROVIDER_SCOPED_RESOURCE_FUNCTION_NAMES: Final = frozenset(
     {

litellm/router_utils/pre_call_checks/continuation_prefill_check.py

@@ -0,0 +1,264 @@
+"""Keeps a mid-stream continuation on deployments whose model supports assistant prefill and whose own params leave
+the output unconstrained."""
+
+from collections.abc import AsyncIterable, AsyncIterator, Mapping, Sequence
+from dataclasses import dataclass
+from itertools import accumulate
+from os.path import commonprefix
+from types import MappingProxyType
+from typing import Final
+
+from pydantic import TypeAdapter, ValidationError
+
+import litellm
+from litellm.integrations.custom_logger import CustomLogger, Span
+from litellm.litellm_core_utils.rules import Rules
+from litellm.router_utils.common_utils import MID_STREAM_CONTINUATION_KWARG
+from litellm.types.llms.openai import AllMessageValues
+from litellm.types.utils import ModelResponse, ModelResponseStream, Usage
+from litellm.utils import supports_assistant_prefill
+
+
+class _ContinuationMarker:
+    """Only the router constructs one, so a JSON request body cannot forge the continuation flag."""
+
+
+MID_STREAM_CONTINUATION_MARKER: Final = _ContinuationMarker()
+
+_STR_KEYED_DICT_ADAPTER: Final = TypeAdapter(dict[str, object])
+_NO_FIELDS: Final[Mapping[str, object]] = MappingProxyType({})
+_UNCONSTRAINED_RESPONSE_FORMAT: Final[Mapping[str, str]] = MappingProxyType({"type": "text"})
+_OUTPUT_CEILING_KEYS: Final = ("max_tokens", "max_completion_tokens")
+
+
+def _str_keyed(value: object) -> Mapping[str, object]:
+    try:
+        return _STR_KEYED_DICT_ADAPTER.validate_python(value)
+    except ValidationError:
+        return _NO_FIELDS
+
+
+def deployment_supports_prefill(deployment: object) -> bool:
+    deployment_map: Final = _str_keyed(deployment)
+    model_info: Final = _str_keyed(deployment_map.get("model_info"))
+    declared: Final = model_info.get("supports_assistant_prefill")
+    if isinstance(declared, bool):
+        return declared
+    litellm_params: Final = _str_keyed(deployment_map.get("litellm_params"))
+    candidates: Final = (model_info.get("base_model"), litellm_params.get("base_model"), litellm_params.get("model"))
+    model: Final = next((candidate for candidate in candidates if isinstance(candidate, str) and candidate), None)
+    return model is not None and supports_assistant_prefill(model=model)
+
+
+def constrains_continuation(params: Mapping[str, object]) -> bool:
+    sent: Final = {**params, **_str_keyed(params.get("extra_body"))}
+    choice_count: Final = sent.get("n")
+    response_format: Final = sent.get("response_format")
+    tool_choice: Final = sent.get("tool_choice")
+    return (
+        (isinstance(choice_count, int) and choice_count > 1)
+        or (response_format is not None and response_format != _UNCONSTRAINED_RESPONSE_FORMAT)
+        or tool_choice == "required"
+        or isinstance(tool_choice, Mapping)
+        or sent.get("merge_reasoning_content_in_choices") is True
+    )
+
+
+def deployment_can_continue(deployment: object) -> bool:
+    litellm_params: Final = _str_keyed(_str_keyed(deployment).get("litellm_params"))
+    return deployment_supports_prefill(deployment) and not constrains_continuation(litellm_params)
+
+
+def continuation_deployments(
+    healthy_deployments: list[dict[str, object]],  # mutable-ok: router deployment-list contract
+    request_kwargs: Mapping[str, object] | None,
+) -> list[dict[str, object]]:  # mutable-ok: returns a mutable deployment list
+    marker: Final = (request_kwargs or {}).get(MID_STREAM_CONTINUATION_KWARG)
+    if not isinstance(marker, _ContinuationMarker):
+        return healthy_deployments
+    return [deployment for deployment in healthy_deployments if deployment_can_continue(deployment)]
+
+
+class ContinuationPrefillDeploymentCheck(CustomLogger):
+    async def async_filter_deployments(
+        self,
+        model: str,
+        healthy_deployments: list[dict[str, object]],  # mutable-ok: CustomLogger deployment-list contract
+        messages: Sequence[AllMessageValues] | None,
+        request_kwargs: Mapping[str, object] | None = None,
+        parent_otel_span: Span | None = None,
+    ) -> list[dict[str, object]]:  # mutable-ok: returns a mutable deployment list
+        return continuation_deployments(healthy_deployments, request_kwargs)
+
+
+def _delta_text(chunk: ModelResponseStream) -> str:
+    if not chunk.choices:
+        return ""
+    content: Final = chunk.choices[0].delta.content
+    return content if isinstance(content, str) else ""
+
+
+_CONTENT_BLOCKS_ADAPTER: Final = TypeAdapter(list[dict[str, object]])
+
+
+def _prefill_text(prefill_message: Mapping[str, object]) -> str:
+    content: Final = prefill_message.get("content")
+    if isinstance(content, str):
+        return content
+    try:
+        blocks: Final = _CONTENT_BLOCKS_ADAPTER.validate_python(content)
+    except ValidationError:
+        return ""
+    return "".join(
+        text for block in blocks if block.get("type") == "text" and isinstance(text := block.get("text"), str)
+    )
+
+
+def _resendable_whitespace(prefill: str) -> str:
+    return prefill[len(prefill.rstrip()) :] if prefill.strip() else ""
+
+
+def continued_text(prefill_message: Mapping[str, object], generated_content: str) -> str | None:
+    prefill: Final = _prefill_text(prefill_message)
+    if generated_content and len(generated_content) < len(prefill) and prefill.startswith(generated_content):
+        return None
+    echo: Final = len(prefill) if prefill and generated_content.startswith(prefill) else 0
+    after_echo: Final = generated_content[echo:]
+    return after_echo[len(commonprefix((after_echo, _resendable_whitespace(prefill)))) :]
+
+
+def _reduced_ceilings(params: Mapping[str, object], emitted_tokens: int) -> Mapping[str, int]:
+    return MappingProxyType(
+        {key: value - emitted_tokens for key in _OUTPUT_CEILING_KEYS if isinstance(value := params.get(key), int)}
+    )
+
+
+def continuation_output_ceilings(
+    request_kwargs: Mapping[str, object], emitted_tokens: int
+) -> Mapping[str, object] | None:
+    """The caller's output ceilings, top-level and in extra_body, minus the tokens already emitted, or None once
+    nothing is left."""
+    extra_body: Final = _str_keyed(request_kwargs.get("extra_body"))
+    top_level: Final = _reduced_ceilings(request_kwargs, emitted_tokens)
+    in_extra_body: Final = _reduced_ceilings(extra_body, emitted_tokens)
+    if any(ceiling <= 0 for ceiling in (*top_level.values(), *in_extra_body.values())):
+        return None
+    if not in_extra_body:
+        return top_level
+    return MappingProxyType({**top_level, "extra_body": {**extra_body, **in_extra_body}})
+
+
+def _with_first_delta(chunk: ModelResponseStream, update: Mapping[str, object]) -> ModelResponseStream:
+    choice: Final = chunk.choices[0]
+    delta: Final = choice.delta.model_copy(update=update)
+    return chunk.model_copy(update={"choices": [choice.model_copy(update={"delta": delta}), *chunk.choices[1:]]})
+
+
+def _without_leading_text(chunk: ModelResponseStream, count: int) -> ModelResponseStream:
+    if count <= 0:
+        return chunk
+    return _with_first_delta(chunk, {"content": _delta_text(chunk)[count:]})
+
+
+def _without_role(chunk: ModelResponseStream) -> ModelResponseStream:
+    if not chunk.choices or chunk.choices[0].delta.role is None:
+        return chunk
+    return _with_first_delta(chunk, {"role": None})
+
+
+def _stream_identity(primary_response: object) -> Mapping[str, object]:
+    if not isinstance(primary_response, ModelResponse):
+        return _NO_FIELDS
+    return MappingProxyType({"id": primary_response.id, "created": primary_response.created})
+
+
+def _drop_leading_text(chunks: tuple[ModelResponseStream, ...], count: int) -> tuple[ModelResponseStream, ...]:
+    lengths: Final = tuple(len(_delta_text(chunk)) for chunk in chunks)
+    offsets: Final = tuple(accumulate(lengths, initial=0))[:-1]
+    return tuple(
+        _without_leading_text(chunk, min(length, count - offset))
+        for chunk, offset, length in zip(chunks, offsets, lengths, strict=True)
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class ContinuedPartial:
+    usage: Usage
+    response_cost: float
+    response_id: str
+
+
+class ContinuationOutput:
+    """Splices a fallback's continuation onto the stream the client already received. Drops a leading echo of the
+    prefill, since some providers (Mistral) return the prefix inside the completion, and the prefill's trailing
+    whitespace sent again by a provider that strips it (Anthropic), keeps the primary's chunk id, created time and single role delta, and
+    runs litellm.post_call_rules over the joined text, since each stream wrapper only sees its own fragment. A
+    continuation nested in another leaves the rules to the outermost one, the only one holding the whole text."""
+
+    def __init__(
+        self,
+        emitted_text: str,
+        prefill_message: Mapping[str, object],
+        model: str,
+        primary_response: object,
+        checks_post_call_rules: bool = True,
+    ) -> None:
+        self._text = emitted_text
+        self._checks_post_call_rules: Final = checks_post_call_rules
+        self._prefill_text: Final = _prefill_text(prefill_message)
+        self._model: Final = model
+        self._identity: Final = _stream_identity(primary_response)
+        self._rules: Final = Rules()
+        self._held: tuple[ModelResponseStream, ...] = ()
+        self._echo_settled = not self._prefill_text
+        self._seam = _resendable_whitespace(self._prefill_text)
+
+    async def splice(self, stream: AsyncIterable[object]) -> AsyncIterator[object]:
+        async for item in stream:
+            if not isinstance(item, ModelResponseStream):
+                yield item
+                continue
+            for released in self._observed(self._past_seam(self._released(item))):
+                yield self._in_primary_stream(released)
+        for released in self._observed(self._past_seam(self._flushed())):
+            yield self._in_primary_stream(released)
+
+    def _in_primary_stream(self, chunk: ModelResponseStream) -> ModelResponseStream:
+        return _without_role(chunk.model_copy(update=self._identity) if self._identity else chunk)
+
+    def _flushed(self) -> tuple[ModelResponseStream, ...]:
+        held: Final = self._held
+        self._held = ()
+        self._echo_settled = True
+        return held
+
+    def _released(self, chunk: ModelResponseStream) -> tuple[ModelResponseStream, ...]:
+        if self._echo_settled:
+            return (chunk,)
+        held: Final = (*self._held, chunk)
+        text: Final = "".join(_delta_text(held_chunk) for held_chunk in held)
+        if len(text) < len(self._prefill_text) and self._prefill_text.startswith(text):
+            self._held = held
+            return ()
+        self._held = ()
+        self._echo_settled = True
+        if not text.startswith(self._prefill_text):
+            return held
+        return _drop_leading_text(held, len(self._prefill_text))
+
+    def _past_seam(self, chunks: tuple[ModelResponseStream, ...]) -> tuple[ModelResponseStream, ...]:
+        return tuple(self._without_seam(chunk) for chunk in chunks)
+
+    def _without_seam(self, chunk: ModelResponseStream) -> ModelResponseStream:
+        text: Final = _delta_text(chunk)
+        matched: Final = len(commonprefix((text, self._seam)))
+        self._seam = self._seam[matched:] if matched == len(text) else ""
+        return _without_leading_text(chunk, matched)
+
+    def _observed(self, chunks: tuple[ModelResponseStream, ...]) -> tuple[ModelResponseStream, ...]:
+        if not self._checks_post_call_rules or not litellm.post_call_rules:
+            return chunks
+        for chunk in chunks:
+            self._text += _delta_text(chunk)
+            self._rules.post_call_rules(input=self._text, model=self._model)
+        return chunks

litellm/types/litellm_params.py

@@ -260,6 +260,7 @@ class PromptOptions:
     prompt_label: str | None = None
     litellm_system_prompt: str | None = None
     custom_prompt_dict: Mapping[str, object] | None = None
+    initial_prompt_value: str | None = None
     roles: Mapping[str, object] | None = None
     final_prompt_value: str | None = None
     bos_token: str | None = None

litellm/types/utils.py

@@ -4045,7 +4045,9 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
 
 DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"})
 
-DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset({"supports_regex_lookaround"})
+DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
+    {"supports_regex_lookaround", "supports_assistant_prefill"}
+)
 
 SHARED_BACKEND_MODEL_INFO_FIELDS: Final[frozenset[str]] = (
     frozenset(ModelInfoBase.__required_keys__ | ModelInfoBase.__optional_keys__)

litellm/utils.py

@@ -89,6 +89,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
     match_capability_generalizations,
     match_fill_missing_generalizations,
 )
+from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
 from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
 from litellm.litellm_core_utils.tokenizer import strip_special_tokens
 from litellm.rust_bridge import tokenizer as tokenizer_dispatch
@@ -3059,6 +3060,20 @@ def supports_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None
     )
 
 
+def supports_assistant_prefill(model: str, custom_llm_provider: str | None = None) -> bool:
+    """
+    Whether the model can continue a prefilled assistant message (Anthropic's
+    ``prefix: True`` trick and equivalents). Missing metadata reads as False,
+    so a mid-stream fallback continuation is only routed to a model known to
+    support it.
+    """
+    return _supports_factory(
+        model=model,
+        custom_llm_provider=custom_llm_provider,
+        key="supports_assistant_prefill",
+    )
+
+
 def supports_thinking_cache_preservation(model: str, custom_llm_provider: str | None = None) -> bool:
     return supports_factory(
         model=model,
@@ -9932,6 +9947,10 @@ class ProviderConfigManager:
             return MinimaxVideoConfig()
         elif LlmProviders.EDENAI == provider:
             return litellm.EdenAIVideoConfig()
+        elif LlmProviders.VOLCENGINE == provider:
+            from litellm.llms.volcengine.videos.transformation import VolcEngineVideoConfig
+
+            return VolcEngineVideoConfig()
         return None
 
     @staticmethod
@@ -9956,6 +9975,7 @@ class ProviderConfigManager:
     def get_provider_realtime_config(
         model: str,
         provider: LlmProviders,
+        drop_params: bool = False,
     ) -> BaseRealtimeConfig | None:
         if LlmProviders.GEMINI == provider:
             from litellm.llms.gemini.realtime.transformation import GeminiRealtimeConfig
@@ -9964,7 +9984,7 @@ class ProviderConfigManager:
         if LlmProviders.META == provider:
             from litellm.llms.meta.realtime.transformation import MetaRealtimeConfig
 
-            return MetaRealtimeConfig()
+            return MetaRealtimeConfig(drop_params=drop_params)
         return None
 
     @staticmethod
@@ -10446,22 +10466,21 @@ def get_non_default_completion_params(kwargs: Mapping[str, object]) -> dict[str,
     return filter_out_litellm_params(kwargs, excluding=litellm.OPENAI_CHAT_COMPLETION_PARAMS)
 
 
-def peek_reasoning_summary_aliases(optional_params: dict) -> object | None:
-    """Read AI-SDK-style reasoning summary from optional_params or nested extra_body.
-
-    Uses key membership (not ``or`` chains) so falsy values like ``""`` are not skipped.
-    """
-    if "reasoningSummary" in optional_params:
-        return optional_params["reasoningSummary"]
-    if "reasoning_summary" in optional_params:
-        return optional_params["reasoning_summary"]
-    extra_body: Final = optional_params.get("extra_body")
-    if isinstance(extra_body, dict):
-        if "reasoningSummary" in extra_body:
-            return extra_body["reasoningSummary"]
-        if "reasoning_summary" in extra_body:
-            return extra_body["reasoning_summary"]
-    return None
+def reasoning_effort_with_summary_alias(
+    reasoning_effort: object, params: Mapping[str, object], accepts_summary: bool
+) -> object:
+    summary: Final = peek_reasoning_summary_aliases(params)
+    if not accepts_summary or not isinstance(summary, str):
+        return reasoning_effort
+    match reasoning_effort:
+        case str():
+            return {"effort": reasoning_effort, "summary": summary}
+        case {"summary": _}:
+            return reasoning_effort
+        case {"effort": str() as effort}:
+            return {"effort": effort, "summary": summary}
+        case _:
+            return reasoning_effort
 
 
 def strip_reasoning_summary_aliases_from_optional_params(

model_prices_and_context_window.json

@@ -2773,7 +2773,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2808,7 +2808,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2843,7 +2843,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2878,7 +2878,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2913,7 +2913,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -2948,7 +2948,7 @@
             "search_context_size_low": 0.01,
             "search_context_size_medium": 0.01
         },
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -3743,7 +3743,7 @@
         "max_tokens": 128000,
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -15494,7 +15494,7 @@
         },
         "supports_adaptive_thinking": true,
         "supports_legacy_thinking": true,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -20832,7 +20832,7 @@
         "output_cost_per_token": 2.5000010000000002e-05,
         "output_dbu_cost_per_token": 0.000357143,
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_legacy_thinking": true,
         "supports_prompt_caching": true,
@@ -21047,7 +21047,7 @@
         "output_cost_per_token": 1.5000020000000002e-05,
         "output_dbu_cost_per_token": 0.000214286,
         "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving",
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_legacy_thinking": true,
         "supports_prompt_caching": true,
@@ -49176,7 +49176,7 @@
         "max_tokens": 64000,
         "mode": "chat",
         "output_cost_per_token": 2.5e-05,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_prompt_caching": true,
@@ -51006,7 +51006,7 @@
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "output_cost_per_token_batches": 7.5e-06,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -51179,7 +51179,7 @@
         "supported_regions": [
             "global"
         ],
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
@@ -51200,7 +51200,7 @@
         "supported_regions": [
             "us-central1"
         ],
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_function_calling": true,
         "supports_prompt_caching": true,
         "supports_reasoning": true,
@@ -58822,7 +58822,7 @@
         "mode": "chat",
         "output_cost_per_token": 1.5e-05,
         "output_cost_per_token_batches": 7.5e-06,
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,
@@ -60467,6 +60467,61 @@
             }
         ]
     },
+    "volcengine/doubao-seedance-2-5-260628": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.216,
+        "output_cost_per_second_480p": 0.096075,
+        "output_cost_per_second_720p": 0.216,
+        "output_cost_per_second_1080p": 0.5346,
+        "supported_modalities": ["text", "image", "video", "audio"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "854x480", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "480x854", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s). List 70 CNY per million tokens at 480p and 720p and 77 at 1080p without a video input, matching Ark's per-resolution table and its 1080p 16:9 example of 3.74 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Accepts up to 50 reference assets (30 images, 10 videos, 10 audios) and clips of 4-30s; per-asset billing beyond the free allowance is not encoded. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
+    "volcengine/doubao-seedance-2-0-260128": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.141942857,
+        "output_cost_per_second_480p": 0.066003429,
+        "output_cost_per_second_720p": 0.141942857,
+        "output_cost_per_second_1080p": 0.354085714,
+        "output_cost_per_second_4k": 0.722057143,
+        "supported_modalities": ["text", "image", "video", "audio"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 46 CNY per million tokens at 480p and 720p, 51 at 1080p, and 26 at 4k without a video input, matching Ark's per-resolution table and its 1080p and 4k 16:9 examples of 2.48 and 5.05 CNY/s, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s; 4k is exclusive to this tier. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
+    "volcengine/doubao-seedance-2-0-fast-260128": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.114171429,
+        "output_cost_per_second_480p": 0.053089714,
+        "output_cost_per_second_720p": 0.114171429,
+        "supported_modalities": ["text", "image", "video", "audio"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 37 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
+    "volcengine/doubao-seedance-2-0-mini-260615": {
+        "litellm_provider": "volcengine",
+        "mode": "video_generation",
+        "source": "https://www.volcengine.com/docs/82379/1544106",
+        "output_cost_per_second": 0.070971429,
+        "output_cost_per_second_480p": 0.033001714,
+        "output_cost_per_second_720p": 0.070971429,
+        "supported_modalities": ["text", "image"],
+        "supported_output_modalities": ["video"],
+        "supported_endpoints": ["/v1/videos"],
+        "provider_specific_entry": {"volcengine_video_frames": {"480p": {"16:9": "864x496", "4:3": "752x560", "1:1": "640x640", "3:4": "560x752", "9:16": "496x864", "21:9": "992x432"}, "720p": {"16:9": "1280x720", "4:3": "1112x834", "1:1": "960x960", "3:4": "834x1112", "9:16": "720x1280", "21:9": "1470x630"}}},
+        "metadata": {"comment": "Ark bills per output token, not per second: tokens = width * height * 24 * seconds / 1024, so a per-second rate per resolution tier is exact for the 16:9 geometry Ark publishes (1280x720 is 21600 tokens/s; the 480p frame is 864x496, 10044 tokens/s, not the tier's nominal 854x480). List 23 CNY per million tokens without a video input, converted at 7.0 CNY/USD like the other volcengine entries. Clips of 4-15s at 480p or 720p. Only text and image inputs are declared; the mini tier's reference video and audio caps were not verified. Requests with a video input bill at a lower rate per token and are rejected, since their cost is not known at create time. provider_specific_entry.volcengine_video_frames is the frame Ark renders per resolution tier and ratio, verified with ffprobe at 480p for every ratio and at 720p for 16:9 and 4:3; the other 720p ratios are from the Seedance 2.0 model card."}
+    },
     "bedrock/us-east-1/zai.glm-5": {
         "input_cost_per_token": 1e-06,
         "output_cost_per_token": 3.2e-06,
@@ -72334,7 +72389,7 @@
         "max_output_tokens": 64000,
         "max_tokens": 64000,
         "mode": "chat",
-        "supports_assistant_prefill": true,
+        "supports_assistant_prefill": false,
         "supports_computer_use": true,
         "supports_function_calling": true,
         "supports_pdf_input": true,

provider_endpoints_support.json

@@ -2766,7 +2766,8 @@
         "batches": false,
         "rerank": false,
         "a2a": true,
-        "interactions": true
+        "interactions": true,
+        "video_generations": true
       }
     },
     "voyage": {

schema.prisma

@@ -1981,6 +1981,7 @@ model LiteLLM_BackgroundInteractionSettlement {
   outcome             String?
 
   @@index([claimed_at], map: "idx_background_interaction_settlement_claimed_at")
+  @@index([settled_at], map: "idx_background_interaction_settlement_settled_at")
 }
 
 model LiteLLM_Lens {

tests/benchmarks/test_prompt_template_benchmarks.py

@@ -0,0 +1,21 @@
+from collections.abc import Mapping
+from typing import Final
+
+import pytest
+
+from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt
+
+LONG_RUN_OF_EMPTY_CACHED_USER_MESSAGES: Final[tuple[Mapping[str, object], ...]] = (
+    {"role": "user", "content": "hi"},
+    {"role": "assistant", "content": "hello"},
+    *({"role": "user", "content": [], "cache_control": {"type": "ephemeral"}} for _ in range(1_000)),
+)
+
+
+@pytest.mark.benchmark
+def test_anthropic_messages_pt_long_run_of_empty_cached_user_messages() -> None:
+    result: Final = anthropic_messages_pt(
+        messages=list(LONG_RUN_OF_EMPTY_CACHED_USER_MESSAGES), model="claude-opus-5", llm_provider="vertex_ai"
+    )
+
+    assert len(result[-1]["content"]) == 1_000

tests/code_coverage_tests/router_code_coverage.py

@@ -114,6 +114,24 @@ ignored_function_names = [
     "_deployment_that_just_failed",  # Tested through the same-boundary hop test in test_router_order_fallback.py
     "_collect_ptu_share_teams",  # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
     "_resolve_team_ptu_ceiling",  # Cached behind team_ptu_ceiling, tested through the live model list test in test_router.py
+    "_register_continuation_prefill_check",  # Tested through the continuation tests in test_router.py
+    "_set_mid_stream_fallback_continuation",  # Tested through the update_settings continuation test in test_router.py
+    "_is_flag_enabled",  # Tested through the string-flag continuation tests in test_router.py
+    "_params_sent_to_deployment",  # Tested through the configured-constraint continuation tests in test_router.py
+    "_names_deployment_that_cannot_continue",  # Tested through the deployment-id continuation tests in test_router.py
+    "_set_retry_policy",  # Tested through update_settings in test_router_retry_policy_update.py
+    "_resolve_pre_call_check_input_tokens",  # Tested through the token-count failure pre-call check tests in test_router.py
+    "_params_neither_dropped_nor_allowed",  # Tested through the allowed-or-dropped param pre-call check tests in test_router.py
+    "_deployment_rejects_params",  # Tested through the unsupported-param pre-call check tests in test_router.py
+    "_deployment_lacks_params",  # Tested through the unsupported-param pre-call check tests in test_router.py
+    "_pre_call_check_requested_chat_params",  # Tested through the unsupported-param pre-call check tests in test_router.py
+    "_deployment_supports_param",  # Tested through the deployment-capability pre-call check tests in test_router.py
+    "_capability_model_and_provider",  # Tested through the authenticating-provider pre-call check test in test_router.py
+    "_exact_base_model_max_input_tokens",  # Tested through get_router_model_info in test_router_model_cost_isolation.py
+    "_base_model_widened_window",  # Tested through get_router_model_info in test_router_model_cost_isolation.py
+    "_provider_model_id",  # Tested through get_router_model_info in test_router_model_cost_isolation.py
+    "_names_route_model",  # Tested through get_router_model_info in test_router_model_cost_isolation.py
+    "_declared_base_model",  # Tested through get_router_model_info in test_router_model_cost_isolation.py
 ]
 
 

tests/integration/_support/upstream.py

@@ -9,9 +9,10 @@ import struct
 import uuid
 import zlib
 from collections import deque
-from collections.abc import AsyncIterator, Mapping
+from collections.abc import AsyncIterator, Callable, Iterator, Mapping
 from dataclasses import dataclass, field
 from queue import SimpleQueue
+from types import MappingProxyType
 from typing import Final, cast
 
 import httpx
@@ -42,7 +43,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile
 from starlette.requests import Request
 from starlette.responses import JSONResponse, Response, StreamingResponse
 from starlette.routing import Route, WebSocketRoute
-from starlette.websockets import WebSocket, WebSocketDisconnect
+from starlette.websockets import WebSocket, WebSocketDisconnect, WebSocketState
 
 JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
 INTERNAL_FIELDS: Final = frozenset(
@@ -81,6 +82,25 @@ class Observation:
     api_key: str = ""
 
 
+@dataclass(frozen=True, slots=True)
+class _RealtimeConnection:
+    websocket: WebSocket
+    response: RealtimeResponse
+    session: dict[str, JsonValue]
+    scenario_id: str
+    authorization: str
+    api_key: str
+    events: Iterator[dict[str, JsonValue]]
+    typed_turn_detection: deque[JsonValue]
+
+    def auto_answers(self) -> bool:
+        return (
+            self.response.dialect == "xai"
+            and len(self.typed_turn_detection) > 0
+            and _is_typed_server_vad(self.typed_turn_detection[-1])
+        )
+
+
 class InteractionState(BaseModel):
     """What the scripted Interactions API answers for one interaction id until a DELETE drops it."""
 
@@ -92,6 +112,12 @@ class InteractionState(BaseModel):
     delay_seconds: float = 0
 
 
+class _ReceivedFrame(BaseModel):
+    model_config = ConfigDict(frozen=True)
+    type: str
+    text: str | None = None
+
+
 class _ScenarioRegistration(BaseModel):
     scenario_id: str
     response: StoredResponse
@@ -415,23 +441,47 @@ class Provider:
     ) -> None:
         await websocket.accept()
         session: Final = _realtime_session(response, scenario_id, websocket.query_params.get("model", ""))
+        frames: Final = _received_texts(websocket)
+        held: Final = await anext(frames, None) if response.created_after_first_frame else None
         for _ in range(response.created_repeats):
             await websocket.send_json({"type": response.created_event, "event_id": _event_id(), "session": session})
-        event_index: Final = iter(response.events)
-        async for message in websocket.iter_json():
-            payload: Final = JSON_OBJECT.validate_python(message)
-            self.observations.put(Observation(websocket.url.path, authorization, payload, "WEBSOCKET_FRAME", api_key))
-            if payload.get("type") == "session.update":
-                await websocket.send_json(
-                    _realtime_update_reply(payload.get("session"), session, response.session_type)
+        connection: Final = _RealtimeConnection(
+            websocket, response, session, scenario_id, authorization, api_key, iter(response.events), deque(maxlen=1)
+        )
+        if held is not None:
+            await self._realtime_frame(connection, held)
+        async for message in frames:
+            await self._realtime_frame(connection, message)
+
+    async def _realtime_frame(self, connection: _RealtimeConnection, message: str) -> None:
+        websocket: Final = connection.websocket
+        response: Final = connection.response
+        payload: Final = JSON_OBJECT.validate_json(message)
+        self.observations.put(
+            Observation(websocket.url.path, connection.authorization, payload, "WEBSOCKET_FRAME", connection.api_key)
+        )
+        frame_type: Final = payload.get("type")
+        if frame_type == "session.update":
+            reply: Final = _realtime_update_reply(
+                payload.get("session"), connection.session, response.session_type, response.dialect
+            )
+            if reply["type"] == "session.updated":
+                connection.typed_turn_detection.extend(
+                    filter(_declares_a_type, _declared_turn_detection(payload.get("session")))
                 )
-                continue
-            if payload.get("type") not in _REALTIME_TRIGGERS:
-                continue
-            event: Final = next(event_index, None)
-            if event is None:
-                continue
-            await websocket.send_json(_rendered_realtime_event(event, scenario_id))
+            await _reply_while_open(websocket, reply)
+            return
+        if frame_type == "response.cancel":
+            await _reply_while_open(websocket, _cancel_not_active(payload.get("event_id"), response.dialect))
+            return
+        if frame_type not in _REALTIME_TRIGGERS:
+            return
+        event: Final = next(connection.events, None)
+        if event is None:
+            return
+        await _reply_while_open(websocket, _rendered_realtime_event(event, connection.scenario_id))
+        if frame_type == "input_audio_buffer.commit" and connection.auto_answers():
+            await _reply_while_open(websocket, _rendered_realtime_event(dict(_AUTO_ANSWER), connection.scenario_id))
 
     async def muse_realtime(self, websocket: WebSocket) -> None:
         await websocket.accept()
@@ -454,8 +504,23 @@ class Provider:
                 await websocket.send_json(_rendered_realtime_event(pending.popleft(), scenario_id))
 
     async def gemini_live(self, websocket: WebSocket) -> None:
-        authorization: Final = websocket.headers.get("authorization", "")
-        scenario_id: Final = websocket.headers.get("x-goog-user-project", "")
+        await self._gemini_live_session(
+            websocket,
+            websocket.headers.get("x-goog-user-project", ""),
+            websocket.headers.get("authorization", ""),
+            _accepts_every_setup,
+        )
+
+    async def gemini_ai_studio_live(self, websocket: WebSocket) -> None:
+        await self._gemini_live_session(websocket, websocket.query_params.get("key", ""), "", _ai_studio_setup_refusal)
+
+    async def _gemini_live_session(
+        self,
+        websocket: WebSocket,
+        scenario_id: str,
+        authorization: str,
+        setup_refusal: Callable[[Mapping[str, JsonValue]], str | None],
+    ) -> None:
         self.observations.put(
             Observation(
                 websocket.url.path,
@@ -477,6 +542,9 @@ class Provider:
                 Observation(websocket.url.path, authorization, payload, "WEBSOCKET_FRAME", scenario_id)
             )
             if "setup" in payload:
+                if (refusal := setup_refusal(payload)) is not None:
+                    await websocket.close(code=_INVALID_ARGUMENT_CLOSE, reason=refusal[:_CLOSE_REASON_BYTES])
+                    return
                 await websocket.send_json({"setupComplete": {}})
                 continue
             if not _GEMINI_LIVE_TRIGGERS.intersection(payload):
@@ -587,6 +655,7 @@ class Provider:
                 WebSocketRoute("/openai/realtime", self.realtime),
                 WebSocketRoute("/v1/asr/realtime", self.muse_realtime),
                 WebSocketRoute(GEMINI_LIVE_PATH, self.gemini_live),
+                WebSocketRoute(GEMINI_AI_STUDIO_LIVE_PATH, self.gemini_ai_studio_live),
             ]
         )
 
@@ -612,6 +681,19 @@ def _interaction_body(interaction_id: str, state: InteractionState) -> dict[str,
 _REALTIME_TRIGGERS: Final = frozenset({"response.create", "input_audio_buffer.commit"})
 _GEMINI_LIVE_TRIGGERS: Final = frozenset({"clientContent", "realtimeInput"})
 GEMINI_LIVE_PATH: Final = "/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent"
+GEMINI_AI_STUDIO_LIVE_PATH: Final = "/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent"
+_INVALID_ARGUMENT_CLOSE: Final = 1007
+_CLOSE_REASON_BYTES: Final = 123
+_AI_STUDIO_TUNING_FIELDS: Final = MappingProxyType(
+    {"prefixPaddingMs": "prefix_padding_ms", "silenceDurationMs": "silence_duration_ms"}
+)
+_AUTO_ANSWER: Final[Mapping[str, JsonValue]] = MappingProxyType(
+    {
+        "type": "response.done",
+        "event_id": "evt_$UNIQUE_ID",
+        "response": {"id": "resp_auto_$UNIQUE_ID", "object": "realtime.response", "status": "completed", "output": []},
+    }
+)
 _TRANSCRIPTION_UPDATE_REFUSED: Final = "Passing a realtime session update to a transcription session is not allowed."
 _REALTIME_UPDATE_REFUSED: Final = "Passing a transcription session update to a realtime session is not allowed."
 _NESTED_TURN_DETECTION_TYPE: Final = "session.audio.input.turn_detection.type"
@@ -651,11 +733,11 @@ def _missing_turn_detection_type(session: Mapping[str, JsonValue]) -> str | None
 
 
 def _realtime_update_reply(
-    session: JsonValue, created: Mapping[str, JsonValue], session_type: str | None
+    session: JsonValue, created: Mapping[str, JsonValue], session_type: str | None, dialect: str
 ) -> dict[str, JsonValue]:
     if not isinstance(session, dict):
         return _realtime_error("invalid_value", "Invalid value for 'session': expected an object.", "session")
-    missing: Final = _missing_turn_detection_type(session)
+    missing: Final = None if dialect == "xai" else _missing_turn_detection_type(session)
     if missing is not None:
         return _realtime_error("missing_required_parameter", f"Missing required parameter: '{missing}'.", missing)
     declared: Final = session.get("type")
@@ -666,6 +748,82 @@ def _realtime_update_reply(
     return {"type": "session.updated", "event_id": _event_id(), "session": {**created, **session}}
 
 
+async def _received_texts(websocket: WebSocket) -> AsyncIterator[str]:
+    while (frame := _ReceivedFrame.model_validate(await websocket.receive())).type == "websocket.receive":
+        if frame.text is not None:
+            yield frame.text
+
+
+async def _reply_while_open(websocket: WebSocket, frame: Mapping[str, JsonValue]) -> None:
+    if websocket.application_state is not WebSocketState.CONNECTED:
+        return
+    try:
+        await websocket.send_json(frame)
+    except WebSocketDisconnect:
+        return
+
+
+def _cancel_not_active(cancel_event_id: JsonValue, dialect: str) -> dict[str, JsonValue]:
+    message: Final = "Cancellation failed: no active response found"
+    if dialect == "xai":
+        own_id: Final = _event_id()
+        return {
+            "type": "error",
+            "event_id": own_id,
+            "error": {
+                "message": message,
+                "type": "invalid_request_error",
+                "code": "invalid_request_error",
+                "event_id": own_id,
+            },
+        }
+    return {
+        "type": "error",
+        "event_id": _event_id(),
+        "error": {
+            "type": "invalid_request_error",
+            "code": "response_cancel_not_active",
+            "message": message,
+            "param": None,
+            "event_id": cancel_event_id,
+        },
+    }
+
+
+def _declared_turn_detection(session: JsonValue) -> tuple[JsonValue, ...]:
+    if not isinstance(session, dict):
+        return ()
+    audio: Final = session.get("audio")
+    audio_input: Final = audio.get("input") if isinstance(audio, dict) else None
+    if isinstance(audio_input, dict) and "turn_detection" in audio_input:
+        return (audio_input["turn_detection"],)
+    return (session["turn_detection"],) if "turn_detection" in session else ()
+
+
+def _declares_a_type(turn_detection: JsonValue) -> bool:
+    return isinstance(turn_detection, dict) and "type" in turn_detection
+
+
+def _is_typed_server_vad(turn_detection: JsonValue) -> bool:
+    return isinstance(turn_detection, dict) and turn_detection.get("type") == "server_vad"
+
+
+def _accepts_every_setup(_payload: Mapping[str, JsonValue]) -> str | None:
+    return None
+
+
+def _ai_studio_setup_refusal(payload: Mapping[str, JsonValue]) -> str | None:
+    setup: Final = payload.get("setup")
+    config: Final = setup.get("realtimeInputConfig") if isinstance(setup, dict) else None
+    detection: Final = config.get("automaticActivityDetection") if isinstance(config, dict) else None
+    if not isinstance(detection, dict) or detection.get("disabled") is not True:
+        return None
+    tuning: Final = tuple(name for key, name in _AI_STUDIO_TUNING_FIELDS.items() if key in detection)
+    if not tuning:
+        return None
+    return f"automatic_activity_detection.disabled is true, but the following fields were also set: {', '.join(tuning)}"
+
+
 def _rendered_realtime_event(event: Mapping[str, JsonValue], scenario_id: str) -> dict[str, JsonValue]:
     return JSON_OBJECT.validate_json(
         json.dumps(event, separators=(",", ":"))

tests/integration/configuration/test_callback_delete.py

@@ -0,0 +1,351 @@
+import json
+import os
+import uuid
+from collections.abc import Generator, Iterator, Mapping
+from contextlib import contextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final
+
+import pytest
+import yaml
+from integration._support.client import JSON_OBJECT, Gateway, JsonValue, eventually, list_value, object_value
+from integration._support.database import read_rows, scratch_database
+from integration._support.process import owned_proxy
+from integration._support.wire import Reply, Request, Wire, wire_server
+
+CALLBACK: Final = "generic_api"
+SIBLING: Final = "langsmith"
+LIST_TYPES: Final = MappingProxyType(
+    {
+        "success_callback": "success",
+        "failure_callback": "failure",
+        "callbacks": "success_and_failure",
+    }
+)
+SINK_ENVIRONMENT: Final = MappingProxyType(
+    {
+        "GENERIC_LOGGER_ENDPOINT": "http://127.0.0.1:9/unused",
+        "LANGSMITH_API_KEY": "synthetic-langsmith-key",
+        "LANGSMITH_BASE_URL": "http://127.0.0.1:9",
+    }
+)
+NOT_FOUND: Final = f"Callback '{CALLBACK}' not found in active configuration"
+WORKER_SYNC_SECONDS: Final = 40
+
+
+@dataclass(frozen=True, slots=True)
+class Rig:
+    proxy: Gateway
+    database_url: str
+
+
+@contextmanager
+def _rig(gateway: Gateway, tmp_path: Path, file_settings: Mapping[str, JsonValue]) -> Generator[Rig]:
+    config: Final = JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
+    path: Final = tmp_path / f"callback_delete_{uuid.uuid4().hex}.yaml"
+    path.write_text(
+        yaml.safe_dump({**config, "litellm_settings": {**object_value(config["litellm_settings"]), **file_settings}})
+    )
+    with (
+        scratch_database() as database_url,
+        owned_proxy(
+            gateway,
+            tmp_path,
+            {"DATABASE_URL": database_url, **SINK_ENVIRONMENT},
+            config=path,
+            remove_environment=("DATABASE_URL_READ_REPLICA",),
+            workers=2,
+        ) as proxy,
+    ):
+        yield Rig(proxy, database_url)
+
+
+@pytest.fixture
+def rig(gateway: Gateway, tmp_path: Path) -> Iterator[Rig]:
+    with _rig(gateway, tmp_path, {}) as created:
+        yield created
+
+
+def _configured(proxy: Gateway) -> tuple[tuple[str, str], ...]:
+    rows: Final = proxy.get("/get/config/callbacks")["callbacks"]
+    assert isinstance(rows, list)
+    return tuple(
+        sorted(
+            (str(row["name"]), str(row["type"]))
+            for row in (object_value(entry) for entry in rows)
+            if not row.get("read_only")
+        )
+    )
+
+
+def _listed(proxy: Gateway, expected: tuple[tuple[str, str], ...]) -> None:
+    eventually(lambda: _configured(proxy), lambda rows: rows == expected, seconds=WORKER_SYNC_SECONDS)
+
+
+def _stored_settings(database_url: str) -> Mapping[str, JsonValue]:
+    rows: Final = read_rows(
+        'SELECT param_value FROM "LiteLLM_Config" WHERE param_name = %s',
+        ("litellm_settings",),
+        database_url=database_url,
+    )
+    assert len(rows) == 1, rows
+    value: Final = rows[0]["param_value"]
+    return object_value(json.loads(value) if isinstance(value, str) else value)
+
+
+def _configure(
+    proxy: Gateway, settings: Mapping[str, JsonValue], environment: Mapping[str, JsonValue] | None = None
+) -> None:
+    body: Final[Mapping[str, JsonValue]] = {
+        "litellm_settings": dict(settings),
+        "environment_variables": None if environment is None else dict(environment),
+    }
+    assert proxy.post("/config/update", body) == {"message": "Config updated successfully"}
+
+
+def _delete(proxy: Gateway, name: str) -> tuple[int, Mapping[str, JsonValue]]:
+    response: Final = proxy.request("POST", "/config/callback/delete", {"callback_name": name})
+    return response.status_code, JSON_OBJECT.validate_json(response.content)
+
+
+@pytest.mark.parametrize("list_key", tuple(LIST_TYPES))
+def test_callback_listed_under_any_settings_list_is_deleted_from_the_listing_and_the_store(
+    rig: Rig, list_key: str
+) -> None:
+    _configure(rig.proxy, {list_key: [CALLBACK]})
+    _listed(rig.proxy, ((CALLBACK, LIST_TYPES[list_key]),))
+
+    status, body = _delete(rig.proxy, CALLBACK)
+
+    assert status == 200, body
+    assert (body["removed_callback"], body["remaining_callbacks"]) == (CALLBACK, [])
+    assert _stored_settings(rig.database_url)[list_key] == []
+    _listed(rig.proxy, ())
+
+
+def test_callback_in_both_success_and_failure_lists_is_removed_from_both_in_one_delete(rig: Rig) -> None:
+    _configure(rig.proxy, {"success_callback": [CALLBACK], "failure_callback": [CALLBACK]})
+    _listed(rig.proxy, ((CALLBACK, "failure"), (CALLBACK, "success")))
+
+    status, body = _delete(rig.proxy, CALLBACK)
+
+    assert status == 200, body
+    stored: Final = _stored_settings(rig.database_url)
+    assert (stored["success_callback"], stored["failure_callback"]) == ([], [])
+    _listed(rig.proxy, ())
+
+
+def test_delete_keeps_other_callbacks_and_reports_what_is_left_in_the_list_it_edited(rig: Rig) -> None:
+    _configure(rig.proxy, {"callbacks": [SIBLING, CALLBACK], "failure_callback": [SIBLING]})
+    _listed(rig.proxy, ((CALLBACK, "success_and_failure"), (SIBLING, "failure"), (SIBLING, "success_and_failure")))
+
+    status, body = _delete(rig.proxy, CALLBACK)
+
+    assert status == 200, body
+    assert body["remaining_callbacks"] == [SIBLING]
+    stored: Final = _stored_settings(rig.database_url)
+    assert (stored["callbacks"], stored["failure_callback"]) == ([SIBLING], [SIBLING])
+    _listed(rig.proxy, ((SIBLING, "failure"), (SIBLING, "success_and_failure")))
+
+
+def test_delete_matches_the_callback_name_case_insensitively(rig: Rig) -> None:
+    _configure(rig.proxy, {"failure_callback": [CALLBACK.upper()], "callbacks": [CALLBACK]})
+    _listed(rig.proxy, ((CALLBACK.upper(), "failure"), (CALLBACK, "success_and_failure")))
+
+    status, body = _delete(rig.proxy, CALLBACK.upper())
+
+    assert status == 200, body
+    assert body["removed_callback"] == CALLBACK
+    _listed(rig.proxy, ())
+
+
+def test_delete_of_an_unconfigured_callback_is_a_404_and_changes_nothing(rig: Rig) -> None:
+    _configure(rig.proxy, {"callbacks": [SIBLING]})
+    _listed(rig.proxy, ((SIBLING, "success_and_failure"),))
+    before: Final = _stored_settings(rig.database_url)
+
+    status, body = _delete(rig.proxy, CALLBACK)
+
+    assert (status, body) == (404, {"detail": {"error": NOT_FOUND}})
+    assert _stored_settings(rig.database_url) == before
+    _listed(rig.proxy, ((SIBLING, "success_and_failure"),))
+
+
+@pytest.mark.parametrize(
+    ("file_key", "database_key"), (("failure_callback", "success_callback"), ("success_callback", "failure_callback"))
+)
+def test_delete_removes_the_database_copy_and_keeps_the_copy_the_config_file_sets(
+    gateway: Gateway, tmp_path: Path, file_key: str, database_key: str
+) -> None:
+    with _rig(gateway, tmp_path, {file_key: [CALLBACK]}) as file_owned:
+        _configure(file_owned.proxy, {database_key: [CALLBACK]})
+        _listed(file_owned.proxy, ((CALLBACK, "failure"), (CALLBACK, "success")))
+
+        status, body = _delete(file_owned.proxy, CALLBACK)
+
+        assert status == 200, body
+        assert body["remaining_callbacks"] == []
+        assert _stored_settings(file_owned.database_url)[database_key] == []
+        _listed(file_owned.proxy, ((CALLBACK, LIST_TYPES[file_key]),))
+
+
+def test_delete_of_a_callback_only_the_config_file_lists_is_refused_and_names_the_file(
+    gateway: Gateway, tmp_path: Path
+) -> None:
+    with _rig(gateway, tmp_path, {"callbacks": [CALLBACK]}) as file_owned:
+        _listed(file_owned.proxy, ((CALLBACK, "success_and_failure"),))
+
+        status, body = _delete(file_owned.proxy, CALLBACK)
+
+        assert status == 400, body
+        detail: Final = object_value(body["detail"])
+        assert (detail["keys"], detail["section"]) == (["callbacks"], "litellm_settings")
+        assert "set in the config file" in str(detail["error"])
+        _listed(file_owned.proxy, ((CALLBACK, "success_and_failure"),))
+
+
+def _completion(request: Request) -> Reply:
+    message: Final = object_value(list_value(JSON_OBJECT.validate_json(request.body)["messages"])[0])
+    if str(message["content"]).endswith("-failure"):
+        return Reply(
+            status=400,
+            body=json.dumps(
+                {"error": {"type": "invalid_request_error", "code": "synthetic", "message": "synthetic failure"}}
+            ).encode(),
+        )
+    return Reply(
+        body=json.dumps(
+            {
+                "id": "chatcmpl-" + uuid.uuid4().hex,
+                "object": "chat.completion",
+                "created": 1,
+                "model": "gpt-4o-mini",
+                "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
+                "usage": {"prompt_tokens": 3, "completion_tokens": 1, "total_tokens": 4},
+            }
+        ).encode()
+    )
+
+
+@dataclass(slots=True)
+class Sink:
+    wire: Wire
+    bodies: list[bytes]  # mutable-ok: drain() consumes the queue, later reads keep earlier batches
+
+    def received(self, markers: tuple[str, ...]) -> bool:
+        self.bodies.extend(request.body for request in self.wire.drain())
+        return all(any(marker.encode() in body for body in self.bodies) for marker in markers)
+
+
+@dataclass(frozen=True, slots=True)
+class Pair:
+    first: Gateway
+    peer: Gateway
+    database_url: str
+    provider: Wire
+    generic: Sink
+    langsmith: Sink
+
+    def running(self, name: str) -> tuple[bool, bool]:
+        return (_runs(self.first, name), _runs(self.peer, name))
+
+
+def _runs(proxy: Gateway, name: str) -> bool:
+    active: Final = list_value(proxy.get("/active/callbacks")["all_litellm_callbacks"])
+    return any(name in str(entry).lower() for entry in active)
+
+
+@contextmanager
+def _pair(gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Generator[Pair]:
+    with (
+        scratch_database() as database_url,
+        wire_server(_completion) as provider,
+        wire_server(lambda _: Reply()) as generic,
+        wire_server(lambda _: Reply()) as langsmith,
+    ):
+        monkeypatch.setenv("DATABASE_URL", database_url)
+        environment: Final = {
+            "DATABASE_URL": database_url,
+            "DEFAULT_FLUSH_INTERVAL_SECONDS": "1",
+            "LITELLM_STORE_AUDIT_LOGS": "True",
+        }
+        removed: Final = (
+            "DATABASE_URL_READ_REPLICA",
+            "GENERIC_LOGGER_ENDPOINT",
+            "GENERIC_LOGGER_HEADERS",
+            "LANGSMITH_API_KEY",
+            "LANGSMITH_BASE_URL",
+        )
+        with (
+            owned_proxy(gateway, tmp_path, environment, remove_environment=removed) as first,
+            owned_proxy(gateway, tmp_path, environment, remove_environment=removed) as peer,
+        ):
+            yield Pair(first, peer, database_url, provider, Sink(generic, []), Sink(langsmith, []))
+
+
+def _send_on_both(pair: Pair, model: str, key: str, marker: str, outcome: str) -> tuple[str, str]:
+    markers: Final = (f"{marker}-first-{outcome}", f"{marker}-peer-{outcome}")
+    for proxy, text in zip((pair.first, pair.peer), markers, strict=True):
+        response = proxy.request(
+            "POST", "/v1/chat/completions", {"model": model, "messages": [{"role": "user", "content": text}]}, key=key
+        )
+        assert response.status_code == (400 if outcome == "failure" else 200), response.text
+    return markers
+
+
+@pytest.mark.parametrize(("list_key", "outcome"), (("failure_callback", "failure"), ("callbacks", "success")))
+def test_deleted_callback_stops_receiving_events_on_both_processes(
+    gateway: Gateway, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, list_key: str, outcome: str
+) -> None:
+    assert os.environ.get("LITELLM_LICENSE"), "LITELLM_LICENSE must be set: audit logs are an enterprise feature"
+    marker: Final = "callback-delete-" + uuid.uuid4().hex
+    with _pair(gateway, tmp_path, monkeypatch) as pair, pair.first.scenario() as scenario:
+        _configure(
+            pair.first,
+            {list_key: [SIBLING, CALLBACK]},
+            {
+                "GENERIC_LOGGER_ENDPOINT": pair.generic.wire.url,
+                "LANGSMITH_API_KEY": "langsmith-key-" + marker,
+                "LANGSMITH_BASE_URL": pair.langsmith.wire.url,
+            },
+        )
+        eventually(lambda: pair.running(SIBLING), lambda running: running == (True, True), seconds=WORKER_SYNC_SECONDS)
+        model: Final = scenario.model(api_base=pair.provider.url + "/v1")
+        key: Final = scenario.key(models=[model])
+        before: Final = _send_on_both(pair, model, key, marker + "-before", outcome)
+        eventually(lambda: pair.generic.received(before), bool, seconds=25)
+        eventually(lambda: pair.langsmith.received(before), bool, seconds=25)
+
+        status, body = _delete(pair.first, SIBLING)
+
+        assert status == 200, body
+        assert (body["removed_callback"], body["remaining_callbacks"]) == (SIBLING, [CALLBACK])
+        assert _stored_settings(pair.database_url)[list_key] == [CALLBACK]
+        audit: Final = eventually(
+            lambda: read_rows(
+                'SELECT before_value, updated_values FROM "LiteLLM_AuditLog" WHERE object_id = %s AND action = %s',
+                ("litellm_settings", "deleted"),
+                database_url=pair.database_url,
+            ),
+            bool,
+            seconds=10,
+        )
+        assert len(audit) == 1, audit
+        before_value: Final = list_value(object_value(audit[0]["before_value"])[list_key])
+        assert sorted(str(name) for name in before_value) == sorted([SIBLING, CALLBACK]), audit
+        assert audit[0]["updated_values"] == {list_key: [CALLBACK]}, audit
+        eventually(
+            lambda: pair.running(SIBLING), lambda running: running == (False, False), seconds=WORKER_SYNC_SECONDS
+        )
+
+        after: Final = _send_on_both(pair, model, key, marker + "-after", outcome)
+        eventually(lambda: pair.generic.received(after), bool, seconds=25)
+        _configure(pair.first, {list_key: [CALLBACK, SIBLING]})
+        eventually(lambda: pair.running(SIBLING), lambda running: running == (True, True), seconds=WORKER_SYNC_SECONDS)
+        barrier: Final = _send_on_both(pair, model, key, marker + "-barrier", outcome)
+        eventually(lambda: pair.langsmith.received(barrier), bool, seconds=25)
+
+        leaked: Final = tuple(text for text in after if any(text.encode() in body for body in pair.langsmith.bodies))
+        assert leaked == (), leaked

tests/integration/cost_calculation/cost_tracking_case.py

@@ -235,6 +235,8 @@ class RealtimeResponse(BaseModel):
     session_type: str | None = None
     created_event: Literal["session.created", "transcription_session.created"] = "session.created"
     created_repeats: int = 1
+    created_after_first_frame: bool = False
+    dialect: Literal["openai", "xai"] = "openai"
 
 
 StoredResponse: TypeAlias = Annotated[

tests/integration/observability/test_bedrock_error_request_id.py

@@ -1,13 +1,208 @@
+import asyncio
+import base64
 import json
 import uuid
-from typing import Final
+from collections.abc import Mapping
+from concurrent.futures import ThreadPoolExecutor
+from types import MappingProxyType
+from typing import Final, Literal, NamedTuple
+from urllib.parse import unquote
 
-from integration._support.client import Gateway, eventually, object_value
+import anthropic
+import httpx
+import openai
+import pytest
+from integration._support.client import Gateway, Scenario, eventually, object_value
 from integration._support.database import read_rows
+from integration._support.responses_stream import (
+    created,
+    delta,
+    error_event,
+    failed,
+    frame,
+    healthy_stream,
+    openai_error_event,
+    spend_row,
+)
+from integration._support.upstream import _aws_event_frame, aws_event_stream_frame
 from integration._support.wire import Reply, Request, wire_server
+from pydantic import JsonValue
+
+import litellm
+from litellm.exceptions import MidStreamFallbackError
+from litellm.litellm_core_utils.litellm_logging import _get_provider_request_id
 
 _MODEL: Final = "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0"
+_BEDROCK_RESPONSES_MODEL: Final = "bedrock/us.openai.gpt-5.6-sol"
 _TOKEN: Final = "synthetic-bedrock-bearer"
+_SHAPES: Final = ("nested-error", "top-level-error", "response.failed", "error-before-output")
+_ERROR_CODES: Final = ("server_error", "rate_limit_exceeded", "invalid_request_error", "content_policy_violation")
+_INVOKE_MODEL_ID: Final = "anthropic.claude-3-haiku-20240307-v1:0"
+_BEDROCK_AWS: Final[Mapping[str, JsonValue]] = MappingProxyType(
+    {
+        "aws_access_key_id": "AKIASCRIPTEDPROVIDER",
+        "aws_secret_access_key": "scripted-secret",
+        "aws_region_name": "us-east-1",
+    }
+)
+_EVENT_STREAM: Final = "application/vnd.amazon.eventstream"
+_BEDROCK_STREAM_TARGETS: Final = frozenset(
+    (
+        "/model/anthropic.claude-sonnet-4-5-20250929-v1:0/converse-stream",
+        f"/model/{_INVOKE_MODEL_ID}/invoke-with-response-stream",
+    )
+)
+
+BedrockFailure = Literal["exception-event", "exception-message", "no-known-event", "truncated-frame"]
+
+
+class _FailureShape(NamedTuple):
+    bedrock_status: int
+    converse_class: str
+    after_output: tuple[bool, ...]
+
+
+BedrockEndpoint = Literal["chat", "responses", "messages", "messages-invoke"]
+_BEDROCK_ENDPOINTS: Final[tuple[BedrockEndpoint, ...]] = ("chat", "responses", "messages", "messages-invoke")
+_BEDROCK_FAILURES: Final[Mapping[BedrockFailure, _FailureShape]] = MappingProxyType(
+    {
+        "exception-event": _FailureShape(500, "ServiceUnavailableError", (False, True)),
+        "exception-message": _FailureShape(429, "RateLimitError", (False, True)),
+        "no-known-event": _FailureShape(502, "BadGatewayError", (False,)),
+        "truncated-frame": _FailureShape(502, "BadGatewayError", (True,)),
+    }
+)
+_BEDROCK_CELLS: Final = tuple(
+    pytest.param(
+        endpoint, failure, after_output, id=f"{endpoint}-{failure}-{'after' if after_output else 'before'}-output"
+    )
+    for endpoint in _BEDROCK_ENDPOINTS
+    for failure, shape in _BEDROCK_FAILURES.items()
+    for after_output in shape.after_output
+)
+
+
+def _stream(identity: str, shape: str, error_code: str, message: str) -> tuple[bytes, ...]:
+    opening: Final = (frame(created(identity)), frame(delta(identity, "partial output")))
+    nested: Final = frame(error_event({"type": error_code, "code": error_code, "message": message}))
+    match shape:
+        case "nested-error":
+            return (*opening, nested)
+        case "top-level-error":
+            return (*opening, frame(openai_error_event(error_code, message)))
+        case "response.failed":
+            return (*opening, frame(failed(identity, error_code, message)))
+        case "error-before-output":
+            return (nested,)
+    raise AssertionError(shape)
+
+
+def _bedrock_frame(event_type: str, payload: dict[str, JsonValue]) -> bytes:
+    return _aws_event_frame(event_type, payload, "sc", "u")
+
+
+def _invoke_chunk(event: dict[str, JsonValue]) -> bytes:
+    return _bedrock_frame("chunk", {"bytes": base64.b64encode(json.dumps(event).encode()).decode()})
+
+
+def _bedrock_output(invoke: bool, text: str = "partial output") -> tuple[bytes, ...]:
+    if not invoke:
+        return (
+            _bedrock_frame("messageStart", {"role": "assistant"}),
+            _bedrock_frame("contentBlockDelta", {"delta": {"text": text}, "contentBlockIndex": 0}),
+        )
+    started: Final[dict[str, JsonValue]] = {
+        "id": f"msg_{uuid.uuid4().hex}",
+        "type": "message",
+        "role": "assistant",
+        "content": [],
+        "model": _INVOKE_MODEL_ID,
+        "stop_reason": None,
+        "usage": {"input_tokens": 3, "output_tokens": 1},
+    }
+    return (
+        _invoke_chunk({"type": "message_start", "message": started}),
+        _invoke_chunk({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}),
+        _invoke_chunk({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}),
+    )
+
+
+def _bedrock_failure_frame(failure: BedrockFailure, message: str) -> bytes:
+    match failure:
+        case "exception-event":
+            return _bedrock_frame("internalServerException", {"message": message})
+        case "exception-message":
+            headers: Final = {
+                ":exception-type": "throttlingException",
+                ":content-type": "application/json",
+                ":message-type": "exception",
+            }
+            return aws_event_stream_frame(headers, json.dumps({"message": message}).encode())
+        case "no-known-event":
+            return _bedrock_frame("somethingBedrockAddedLater", {"message": message})
+        case "truncated-frame":
+            return _bedrock_frame("contentBlockDelta", {"delta": {"text": message}, "contentBlockIndex": 0})[:-7]
+
+
+def _bedrock_failure_text(failure: BedrockFailure, message: str) -> str:
+    return "undecoded bytes after" if failure == "truncated-frame" else message
+
+
+def _bedrock_stream_request(
+    endpoint: BedrockEndpoint, model: str, prompt: str, *, stream: bool = True
+) -> tuple[str, dict[str, JsonValue]]:
+    extra: Final[dict[str, JsonValue]] = {"stream": stream, "num_retries": 0, "cache": {"no-cache": True}}
+    turn: Final[list[JsonValue]] = [{"role": "user", "content": prompt}]
+    match endpoint:
+        case "chat":
+            return "/v1/chat/completions", {"model": model, "messages": turn, **extra}
+        case "responses":
+            return "/v1/responses", {"model": model, "input": prompt, **extra}
+        case "messages" | "messages-invoke":
+            return "/v1/messages", {"model": model, "messages": turn, "max_tokens": 16, **extra}
+
+
+def _failure_information(call_id: str) -> dict[str, JsonValue]:
+    row: Final = spend_row(call_id)
+    assert row["status"] == "failure", row
+    metadata: Final = row["metadata"]
+    parsed: Final = json.loads(metadata) if isinstance(metadata, str) else object_value(metadata)
+    return object_value(parsed["error_information"])
+
+
+def _streamed_response(gateway: Gateway, model: str, prompt: str) -> httpx.Response:
+    response: Final = gateway.request(
+        "POST",
+        "/v1/responses",
+        {"model": model, "input": prompt, "stream": True, "num_retries": 0, "cache": {"no-cache": True}},
+    )
+    assert response.status_code == 200, response.text
+    return response
+
+
+async def _consume_sdk_stream(asynchronous: bool, api_base: str, identity: str) -> None:
+    if asynchronous:
+        async_stream: Final = await litellm.aresponses(
+            model="openai/gpt-6",
+            input=identity,
+            stream=True,
+            api_base=api_base,
+            api_key="synthetic-provider-key",
+            num_retries=0,
+        )
+        async for _ in async_stream:
+            pass
+        return
+    tuple(
+        litellm.responses(
+            model="openai/gpt-6",
+            input=identity,
+            stream=True,
+            api_base=api_base,
+            api_key="synthetic-provider-key",
+            num_retries=0,
+        )
+    )
 
 
 def test_bedrock_500_keeps_amzn_request_id_on_error_headers_and_failure_log(gateway: Gateway) -> None:
@@ -54,3 +249,556 @@ def test_bedrock_500_keeps_amzn_request_id_on_error_headers_and_failure_log(gate
         parsed: Final = json.loads(metadata) if isinstance(metadata, str) else object_value(metadata)
         error_information: Final = object_value(parsed["error_information"])
         assert error_information["error_provider_request_id"] == amzn_request_id, error_information
+
+
+@pytest.mark.parametrize("shape", _SHAPES)
+@pytest.mark.parametrize("error_code", _ERROR_CODES)
+def test_responses_stream_failure_log_retains_provider_request_id(
+    gateway: Gateway, shape: str, error_code: str
+) -> None:
+    identity: Final = f"resp_{uuid.uuid4().hex}"
+    provider_request_id: Final = str(uuid.uuid4())
+    message: Final = f"stream failure {identity}"
+
+    def respond(request: Request) -> Reply:
+        assert request.method == "POST" and request.target == "/responses", request.target
+        assert json.loads(request.body)["stream"] is True
+        return Reply(
+            headers={"x-amzn-RequestId": provider_request_id},
+            content_type="text/event-stream",
+            chunks=_stream(identity, shape, error_code, message),
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model="openai/gpt-6", api_base=wire.url, num_retries=0)
+        response: Final = _streamed_response(gateway, model, identity)
+        assert message in response.text, response.text
+        assert ("partial output" in response.text) is (shape != "error-before-output"), response.text
+        assert response.headers.get("llm_provider-x-amzn-requestid") == provider_request_id, dict(response.headers)
+        assert len(wire.drain()) == 1
+        information: Final = _failure_information(response.headers["x-litellm-call-id"])
+        assert information["error_provider_request_id"] == provider_request_id, information
+
+
+@pytest.mark.parametrize("shape", _SHAPES)
+def test_responses_stream_failure_without_provider_request_id_logs_none(gateway: Gateway, shape: str) -> None:
+    identity: Final = f"resp_{uuid.uuid4().hex}"
+    message: Final = f"stream failure {identity}"
+
+    def respond(request: Request) -> Reply:
+        assert request.method == "POST" and request.target == "/responses", request.target
+        return Reply(content_type="text/event-stream", chunks=_stream(identity, shape, "server_error", message))
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model="openai/gpt-6", api_base=wire.url, num_retries=0)
+        response: Final = _streamed_response(gateway, model, identity)
+        assert message in response.text, response.text
+        assert "llm_provider-x-amzn-requestid" not in response.headers, dict(response.headers)
+        assert len(wire.drain()) == 1
+        information: Final = _failure_information(response.headers["x-litellm-call-id"])
+        assert information["error_class"] == "InternalServerError", information
+        assert information["error_provider_request_id"] is None, information
+
+
+@pytest.mark.parametrize("shape", ("nested-error", "response.failed"))
+def test_bedrock_responses_stream_server_error_logs_amzn_request_id(gateway: Gateway, shape: str) -> None:
+    identity: Final = f"resp_{uuid.uuid4().hex}"
+    amzn_request_id: Final = str(uuid.uuid4())
+    message: Final = f"bedrock stream failure {identity}"
+
+    def respond(request: Request) -> Reply:
+        assert request.method == "POST" and request.target == "/openai/v1/responses", request.target
+        assert request.headers["authorization"] == f"Bearer {_TOKEN}", request.headers
+        assert json.loads(request.body)["stream"] is True
+        return Reply(
+            headers={"x-amzn-RequestId": amzn_request_id},
+            content_type="text/event-stream",
+            chunks=_stream(identity, shape, "server_error", message),
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model=_BEDROCK_RESPONSES_MODEL,
+            api_key=_TOKEN,
+            api_base=wire.url,
+            aws_region_name="us-east-1",
+            num_retries=0,
+        )
+        response: Final = _streamed_response(gateway, model, identity)
+        assert "partial output" in response.text and message in response.text, response.text
+        assert response.headers.get("llm_provider-x-amzn-requestid") == amzn_request_id, dict(response.headers)
+        assert len(wire.drain()) == 1
+        information: Final = _failure_information(response.headers["x-litellm-call-id"])
+        assert information["llm_provider"] == "bedrock", information
+        assert information["error_provider_request_id"] == amzn_request_id, information
+
+
+def test_responses_stream_failure_log_reads_openai_x_request_id(gateway: Gateway) -> None:
+    identity: Final = f"resp_{uuid.uuid4().hex}"
+    provider_request_id: Final = f"req_{uuid.uuid4().hex}"
+    message: Final = f"stream failure {identity}"
+
+    def respond(request: Request) -> Reply:
+        assert request.method == "POST" and request.target == "/responses", request.target
+        return Reply(
+            headers={"x-request-id": provider_request_id},
+            content_type="text/event-stream",
+            chunks=_stream(identity, "top-level-error", "server_error", message),
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model="openai/gpt-6", api_base=wire.url, num_retries=0)
+        response: Final = _streamed_response(gateway, model, identity)
+        assert message in response.text, response.text
+        assert len(wire.drain()) == 1
+        information: Final = _failure_information(response.headers["x-litellm-call-id"])
+        assert information["error_provider_request_id"] == provider_request_id, information
+
+
+@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async"))
+@pytest.mark.parametrize("event_type", ("error", "response.failed"))
+async def test_responses_sdk_stream_error_retains_provider_request_id(asynchronous: bool, event_type: str) -> None:
+    identity: Final = f"resp_{uuid.uuid4().hex}"
+    provider_request_id: Final = str(uuid.uuid4())
+    shape: Final = "nested-error" if event_type == "error" else "response.failed"
+
+    def respond(request: Request) -> Reply:
+        assert request.method == "POST" and request.target == "/responses", request.target
+        return Reply(
+            headers={"x-amzn-RequestId": provider_request_id, "retry-after": "7"},
+            content_type="text/event-stream",
+            chunks=_stream(identity, shape, "server_error", identity),
+        )
+
+    with wire_server(respond) as wire:
+        with pytest.raises(MidStreamFallbackError) as raised:
+            await _consume_sdk_stream(asynchronous, wire.url, identity)
+        assert raised.value.status_code == 500
+        assert raised.value.response.status_code == 500
+        assert str(raised.value.response.request.url) == "https://openai.com/v1/"
+        assert raised.value.generated_content == "partial output"
+        assert dict(raised.value.response.headers) == {"x-amzn-requestid": provider_request_id}
+        assert _get_provider_request_id(raised.value) == provider_request_id
+        assert raised.value.original_exception is not None
+        assert _get_provider_request_id(raised.value.original_exception) == provider_request_id
+        assert len(wire.drain()) == 1
+
+
+def _concurrent_reply(request: Request, failing: tuple[str, ...], recovery: str) -> Reply:
+    prompt: Final = json.loads(request.body)["input"]
+    assert prompt in (*failing, recovery)
+    chunks: Final = (
+        healthy_stream(f"resp_{prompt}", "recovered")
+        if prompt == recovery
+        else _stream(f"resp_{prompt}", "nested-error", "server_error", prompt)
+    )
+    return Reply(content_type="text/event-stream", headers={"x-amzn-requestid": prompt}, chunks=chunks)
+
+
+def test_concurrent_responses_failures_keep_distinct_request_ids_and_recover(gateway: Gateway) -> None:
+    identity: Final = uuid.uuid4().hex
+    prompts: Final = tuple(f"{identity}-{index}" for index in range(4))
+    recovery: Final = f"{identity}-recovery"
+
+    with (
+        wire_server(lambda request: _concurrent_reply(request, prompts, recovery)) as wire,
+        gateway.scenario() as scenario,
+    ):
+        model: Final = scenario.model(model="openai/gpt-6", api_base=wire.url, num_retries=0)
+
+        def call(prompt: str) -> str:
+            response: Final = _streamed_response(gateway, model, prompt)
+            assert (prompt if prompt != recovery else "recovered") in response.text, response.text
+            return response.headers["x-litellm-call-id"]
+
+        with ThreadPoolExecutor(max_workers=4) as workers:
+            call_ids: Final = tuple(workers.map(call, prompts))
+        recovered_call_id: Final = call(recovery)
+        assert len(wire.drain()) == 5
+        for prompt, call_id in zip(prompts, call_ids, strict=True):
+            assert _failure_information(call_id)["error_provider_request_id"] == prompt
+        assert spend_row(recovered_call_id)["status"] == "success"
+
+
+def _bedrock_logged_error(endpoint: BedrockEndpoint, failure: BedrockFailure) -> tuple[str, str]:
+    shape: Final = _BEDROCK_FAILURES[failure]
+    if endpoint == "messages-invoke":
+        return "BedrockError", str(shape.bedrock_status)
+    return shape.converse_class, "503" if failure == "exception-event" else str(shape.bedrock_status)
+
+
+def _bedrock_client_status(endpoint: BedrockEndpoint, failure: BedrockFailure, after_output: bool) -> int:
+    logged_code: Final = int(_bedrock_logged_error(endpoint, failure)[1])
+    if after_output or endpoint in ("responses", "messages"):
+        return 200
+    if endpoint == "messages-invoke" and failure == "no-known-event":
+        return 200
+    return logged_code
+
+
+@pytest.mark.parametrize(("endpoint", "failure", "after_output"), _BEDROCK_CELLS)
+def test_bedrock_event_stream_failure_log_retains_amzn_request_id(
+    gateway: Gateway, endpoint: BedrockEndpoint, failure: BedrockFailure, after_output: bool
+) -> None:
+    amzn_request_id: Final = str(uuid.uuid4())
+    message: Final = f"bedrock stream failure {amzn_request_id}"
+    invoke: Final = endpoint == "messages-invoke"
+    frames: Final = (*(_bedrock_output(invoke) if after_output else ()), _bedrock_failure_frame(failure, message))
+
+    def respond(request: Request) -> Reply:
+        assert request.method == "POST" and unquote(request.target) in _BEDROCK_STREAM_TARGETS, request.target
+        return Reply(
+            headers={"x-amzn-RequestId": amzn_request_id, "retry-after": "7"},
+            content_type=_EVENT_STREAM,
+            body=b"".join(frames),
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model=f"bedrock/invoke/{_INVOKE_MODEL_ID}" if invoke else _MODEL,
+            api_base=wire.url,
+            num_retries=0,
+            **_BEDROCK_AWS,
+        )
+        path, body = _bedrock_stream_request(endpoint, model, message)
+        with gateway.client.stream(
+            "POST", path, json=body, headers={"Authorization": f"Bearer {gateway.key}"}
+        ) as response:
+            text: Final = "\n".join(line for line in response.iter_lines() if line)
+            status: Final = response.status_code
+            response_headers: Final = dict(response.headers)
+        assert status == _bedrock_client_status(endpoint, failure, after_output), text
+        assert _bedrock_failure_text(failure, message) in text and ("partial output" in text) is after_output, text
+        assert len(wire.drain()) == 1
+        information: Final = _failure_information(response_headers["x-litellm-call-id"])
+        assert (information["error_class"], information["error_code"]) == _bedrock_logged_error(endpoint, failure)
+        assert information["error_provider_request_id"] == amzn_request_id, information
+        if status != 200 and not invoke:
+            assert response_headers.get("llm_provider-x-amzn-requestid") == amzn_request_id, response_headers
+            assert "llm_provider-retry-after" not in response_headers, response_headers
+
+
+def _bedrock_model(scenario: Scenario, endpoint: BedrockEndpoint, api_base: str) -> str:
+    return scenario.model(
+        model=f"bedrock/invoke/{_INVOKE_MODEL_ID}" if endpoint == "messages-invoke" else _MODEL,
+        api_base=api_base,
+        num_retries=0,
+        **_BEDROCK_AWS,
+    )
+
+
+def _bedrock_stream(
+    gateway: Gateway, endpoint: BedrockEndpoint, model: str, prompt: str
+) -> tuple[int, str, dict[str, str]]:
+    path, body = _bedrock_stream_request(endpoint, model, prompt)
+    with gateway.client.stream("POST", path, json=body, headers={"Authorization": f"Bearer {gateway.key}"}) as response:
+        text: Final = "\n".join(line for line in response.iter_lines() if line)
+        return response.status_code, text, dict(response.headers)
+
+
+@pytest.mark.parametrize("after_output", (False, True), ids=("before-output", "after-output"))
+@pytest.mark.parametrize("endpoint", _BEDROCK_ENDPOINTS)
+def test_bedrock_event_stream_failure_without_amzn_request_id_logs_none(
+    gateway: Gateway, endpoint: BedrockEndpoint, after_output: bool
+) -> None:
+    message: Final = f"bedrock stream failure {uuid.uuid4()}"
+    frames: Final = (
+        *(_bedrock_output(endpoint == "messages-invoke") if after_output else ()),
+        _bedrock_failure_frame("exception-event", message),
+    )
+
+    def respond(request: Request) -> Reply:
+        assert unquote(request.target) in _BEDROCK_STREAM_TARGETS, request.target
+        return Reply(headers={"retry-after": "7"}, content_type=_EVENT_STREAM, body=b"".join(frames))
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        status, text, response_headers = _bedrock_stream(
+            gateway, endpoint, _bedrock_model(scenario, endpoint, wire.url), message
+        )
+        assert status == _bedrock_client_status(endpoint, "exception-event", after_output), text
+        assert message in text, text
+        assert not any("amzn" in name for name in response_headers), response_headers
+        assert len(wire.drain()) == 1
+        information: Final = _failure_information(response_headers["x-litellm-call-id"])
+        assert (information["error_class"], information["error_code"]) == _bedrock_logged_error(
+            endpoint, "exception-event"
+        )
+        assert information["error_provider_request_id"] is None, information
+
+
+@pytest.mark.parametrize("endpoint", ("responses", "messages", "messages-invoke"))
+def test_bedrock_non_stream_500_logs_amzn_request_id_on_every_endpoint(
+    gateway: Gateway, endpoint: BedrockEndpoint
+) -> None:
+    amzn_request_id: Final = str(uuid.uuid4())
+    targets: Final = frozenset(
+        (
+            "/model/anthropic.claude-sonnet-4-5-20250929-v1:0/converse",
+            f"/model/{_INVOKE_MODEL_ID}/invoke",
+        )
+    )
+
+    def respond(request: Request) -> Reply:
+        assert request.method == "POST" and unquote(request.target) in targets, request.target
+        return Reply(
+            status=500,
+            headers={"x-amzn-RequestId": amzn_request_id},
+            body=b'{"message":"synthetic bedrock failure"}',
+        )
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        path, body = _bedrock_stream_request(
+            endpoint, _bedrock_model(scenario, endpoint, wire.url), amzn_request_id, stream=False
+        )
+        response: Final = gateway.request("POST", path, body)
+        assert response.status_code == 503, response.text
+        if endpoint == "responses":
+            assert response.headers.get("llm_provider-x-amzn-requestid") == amzn_request_id, dict(response.headers)
+        assert len(wire.drain()) == 1
+        information: Final = _failure_information(response.headers["x-litellm-call-id"])
+        assert information["error_provider_request_id"] == amzn_request_id, information
+
+
+@pytest.mark.parametrize("amzn_request_id", ("", "a" * 5000, "dup-1, dup-1"), ids=("empty", "5kb", "joined-duplicate"))
+def test_bedrock_stream_failure_logs_odd_request_ids_like_the_non_stream_error(
+    gateway: Gateway, amzn_request_id: str
+) -> None:
+    stream_body: Final = b"".join((*_bedrock_output(False), _bedrock_failure_frame("exception-event", "odd id")))
+
+    def respond(request: Request) -> Reply:
+        if unquote(request.target) in _BEDROCK_STREAM_TARGETS:
+            return Reply(headers={"x-amzn-RequestId": amzn_request_id}, content_type=_EVENT_STREAM, body=stream_body)
+        return Reply(status=500, headers={"x-amzn-RequestId": amzn_request_id}, body=b'{"message":"odd id"}')
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = _bedrock_model(scenario, "chat", wire.url)
+        status, text, stream_headers = _bedrock_stream(gateway, "chat", model, "odd id")
+        path, body = _bedrock_stream_request("chat", model, "odd id", stream=False)
+        non_stream: Final = gateway.request("POST", path, body)
+        assert status == 200 and "odd id" in text, text
+        assert non_stream.status_code == 503, non_stream.text
+        assert len(wire.drain()) == 2
+        streamed: Final = _failure_information(stream_headers["x-litellm-call-id"])
+        unstreamed: Final = _failure_information(non_stream.headers["x-litellm-call-id"])
+        assert streamed["error_provider_request_id"] == unstreamed["error_provider_request_id"], (streamed, unstreamed)
+
+
+_BURST_CALLS: Final = tuple(
+    (endpoint, failure, after_output)
+    for endpoint in _BEDROCK_ENDPOINTS
+    for failure, shape in _BEDROCK_FAILURES.items()
+    for after_output in shape.after_output
+)
+
+
+def _healthy_bedrock_stream(invoke: bool) -> bytes:
+    closing: Final = (
+        (
+            _invoke_chunk({"type": "content_block_stop", "index": 0}),
+            _invoke_chunk(
+                {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 1}}
+            ),
+            _invoke_chunk({"type": "message_stop"}),
+        )
+        if invoke
+        else (
+            _bedrock_frame("contentBlockStop", {"contentBlockIndex": 0}),
+            _bedrock_frame("messageStop", {"stopReason": "end_turn"}),
+            _bedrock_frame("metadata", {"usage": {"inputTokens": 3, "outputTokens": 1, "totalTokens": 4}}),
+        )
+    )
+    return b"".join((*_bedrock_output(invoke, "recovered"), *closing))
+
+
+def test_concurrent_bedrock_stream_failures_on_every_endpoint_keep_distinct_request_ids(gateway: Gateway) -> None:
+    identity: Final = uuid.uuid4().hex
+    failing: Final = {f"{identity}-{index:02d}": call for index, call in enumerate(_BURST_CALLS)}
+    recoveries: Final = {f"{identity}-r{index}": endpoint for index, endpoint in enumerate(_BEDROCK_ENDPOINTS)}
+
+    def respond(request: Request) -> Reply:
+        (token,) = (token for token in (*failing, *recoveries) if token in request.body.decode())
+        if token in recoveries:
+            body = _healthy_bedrock_stream(recoveries[token] == "messages-invoke")
+        else:
+            endpoint, failure, after_output = failing[token]
+            opening = _bedrock_output(endpoint == "messages-invoke") if after_output else ()
+            body = b"".join((*opening, _bedrock_failure_frame(failure, token)))
+        return Reply(headers={"x-amzn-RequestId": token, "retry-after": "7"}, content_type=_EVENT_STREAM, body=body)
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        models: Final = {endpoint: _bedrock_model(scenario, endpoint, wire.url) for endpoint in _BEDROCK_ENDPOINTS}
+
+        def call(token: str, endpoint: BedrockEndpoint) -> tuple[int, str, dict[str, str]]:
+            return _bedrock_stream(gateway, endpoint, models[endpoint], token)
+
+        with ThreadPoolExecutor(max_workers=8) as workers:
+            outcomes: Final = dict(
+                zip(failing, workers.map(call, failing, (endpoint for endpoint, _, _ in failing.values())), strict=True)
+            )
+        recovered: Final = {token: call(token, endpoint) for token, endpoint in recoveries.items()}
+        assert len(wire.drain()) == len(failing) + len(recoveries)
+        for token, (endpoint, failure, after_output) in failing.items():
+            status, text, response_headers = outcomes[token]
+            assert status == _bedrock_client_status(endpoint, failure, after_output), (token, text)
+            assert _bedrock_failure_text(failure, token) in text, (token, text)
+            information = _failure_information(response_headers["x-litellm-call-id"])
+            assert information["error_provider_request_id"] == token, (token, information)
+        for token, (status, text, response_headers) in recovered.items():
+            assert status == 200 and "recovered" in text, (token, text)
+            assert spend_row(response_headers["x-litellm-call-id"])["status"] == "success"
+
+
+def _sdk_stream_text(
+    client_kind: str, asynchronous: bool, gateway: Gateway, model: str, prompt: str
+) -> tuple[str, str]:
+    extra_body: Final[dict[str, JsonValue]] = {"num_retries": 0, "cache": {"no-cache": True}}
+    turn: Final = [{"role": "user", "content": prompt}]
+
+    def open_stream(client: openai.OpenAI | openai.AsyncOpenAI | anthropic.Anthropic | anthropic.AsyncAnthropic):
+        match client_kind:
+            case "openai-chat":
+                return client.chat.completions.with_raw_response.create(
+                    model=model, messages=turn, stream=True, extra_body=extra_body
+                )
+            case "openai-responses":
+                return client.responses.with_raw_response.create(
+                    model=model, input=prompt, stream=True, extra_body=extra_body
+                )
+            case "anthropic-messages":
+                return client.messages.with_raw_response.create(
+                    model=model, messages=turn, max_tokens=16, stream=True, extra_body=extra_body
+                )
+        raise AssertionError(client_kind)
+
+    seen: Final[list[str]] = []
+    base_url: Final = str(gateway.client.base_url).rstrip("/")
+    if not asynchronous:
+        with httpx.Client(timeout=15, trust_env=False) as transport:
+            client = (
+                anthropic.Anthropic(api_key=gateway.key, base_url=base_url, max_retries=0, http_client=transport)
+                if client_kind == "anthropic-messages"
+                else openai.OpenAI(api_key=gateway.key, base_url=f"{base_url}/v1", max_retries=0, http_client=transport)
+            )
+            raw = open_stream(client)
+            try:
+                seen.extend(repr(event) for event in raw.parse())
+            except (openai.APIError, anthropic.APIError) as error:
+                seen.append(str(error))
+            return raw.headers["x-litellm-call-id"], "\n".join(seen)
+
+    async def run() -> str:
+        async with httpx.AsyncClient(timeout=15, trust_env=False) as transport:
+            client = (
+                anthropic.AsyncAnthropic(api_key=gateway.key, base_url=base_url, max_retries=0, http_client=transport)
+                if client_kind == "anthropic-messages"
+                else openai.AsyncOpenAI(
+                    api_key=gateway.key, base_url=f"{base_url}/v1", max_retries=0, http_client=transport
+                )
+            )
+            raw = await open_stream(client)
+            try:
+                async for event in raw.parse():
+                    seen.append(repr(event))
+            except (openai.APIError, anthropic.APIError) as error:
+                seen.append(str(error))
+            return raw.headers["x-litellm-call-id"]
+
+    return asyncio.run(run()), "\n".join(seen)
+
+
+@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async"))
+@pytest.mark.parametrize("client_kind", ("openai-chat", "openai-responses", "anthropic-messages"))
+def test_sdk_clients_streaming_bedrock_failure_through_proxy_log_amzn_request_id(
+    gateway: Gateway, client_kind: str, asynchronous: bool
+) -> None:
+    amzn_request_id: Final = str(uuid.uuid4())
+    message: Final = f"bedrock stream failure {amzn_request_id}"
+    frames: Final = (*_bedrock_output(False), _bedrock_failure_frame("exception-event", message))
+
+    def respond(request: Request) -> Reply:
+        assert unquote(request.target) in _BEDROCK_STREAM_TARGETS, request.target
+        return Reply(headers={"x-amzn-RequestId": amzn_request_id}, content_type=_EVENT_STREAM, body=b"".join(frames))
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        call_id, seen = _sdk_stream_text(
+            client_kind, asynchronous, gateway, _bedrock_model(scenario, "chat", wire.url), message
+        )
+        assert "partial output" in seen and message in seen, seen
+        assert len(wire.drain()) == 1
+        information: Final = _failure_information(call_id)
+        assert information["error_provider_request_id"] == amzn_request_id, information
+
+
+async def _consume_bedrock_sdk_stream(api: Literal["chat", "responses"], asynchronous: bool, api_base: str) -> str:
+    common: Final[dict[str, JsonValue]] = {
+        "model": _MODEL,
+        "stream": True,
+        "api_base": api_base,
+        "num_retries": 0,
+        **_BEDROCK_AWS,
+    }
+    turn: Final[list[JsonValue]] = [{"role": "user", "content": "probe"}]
+    match api, asynchronous:
+        case "chat", True:
+            return "".join([repr(chunk) async for chunk in await litellm.acompletion(messages=turn, **common)])
+        case "chat", False:
+            return "".join(repr(chunk) for chunk in litellm.completion(messages=turn, **common))
+        case "responses", True:
+            return "".join([repr(event) async for event in await litellm.aresponses(input="probe", **common)])
+        case "responses", False:
+            return "".join(repr(event) for event in litellm.responses(input="probe", **common))
+
+
+@pytest.mark.parametrize("asynchronous", (False, True), ids=("sync", "async"))
+@pytest.mark.parametrize("api", ("chat", "responses"))
+async def test_bedrock_sdk_stream_exception_carries_amzn_request_id(
+    api: Literal["chat", "responses"], asynchronous: bool
+) -> None:
+    amzn_request_id: Final = str(uuid.uuid4())
+    message: Final = f"bedrock stream failure {uuid.uuid4()}"
+    frames: Final = (*_bedrock_output(False), _bedrock_failure_frame("exception-event", message))
+
+    def respond(request: Request) -> Reply:
+        assert unquote(request.target) in _BEDROCK_STREAM_TARGETS, request.target
+        return Reply(
+            headers={"x-amzn-RequestId": amzn_request_id, "retry-after": "7"},
+            content_type=_EVENT_STREAM,
+            body=b"".join(frames),
+        )
+
+    with wire_server(respond) as wire:
+        with pytest.raises(MidStreamFallbackError) as raised:
+            await _consume_bedrock_sdk_stream(api, asynchronous, wire.url)
+        assert len(wire.drain()) == 1
+    provider_error: Final = raised.value.original_exception
+    assert isinstance(provider_error, litellm.ServiceUnavailableError), provider_error
+    assert raised.value.generated_content == "partial output"
+    assert message in str(provider_error)
+    assert _get_provider_request_id(provider_error) == amzn_request_id
+    assert "retry-after" not in provider_error.response.headers, provider_error.response.headers
+
+
+@pytest.mark.parametrize("api", ("chat", "responses"))
+@pytest.mark.parametrize("healthy", (True, False), ids=("healthy", "failing"))
+async def test_sync_bedrock_sdk_stream_with_non_ascii_request_id_header_keeps_streaming(
+    api: Literal["chat", "responses"], healthy: bool
+) -> None:
+    body: Final = (
+        _healthy_bedrock_stream(False)
+        if healthy
+        else b"".join((*_bedrock_output(False), _bedrock_failure_frame("exception-event", "non-ascii id failure")))
+    )
+
+    def respond(request: Request) -> Reply:
+        assert unquote(request.target) in _BEDROCK_STREAM_TARGETS, request.target
+        return Reply(headers={"x-amzn-RequestId": "req-\u00fc-1"}, content_type=_EVENT_STREAM, body=body)
+
+    with wire_server(respond) as wire:
+        if healthy:
+            assert "recovered" in await _consume_bedrock_sdk_stream(api, False, wire.url)
+        else:
+            with pytest.raises(MidStreamFallbackError) as raised:
+                await _consume_bedrock_sdk_stream(api, False, wire.url)
+            assert "non-ascii id failure" in str(raised.value.original_exception)
+            assert _get_provider_request_id(raised.value.original_exception) is None
+        assert len(wire.drain()) == 1

tests/integration/observability/test_content_filter_logging_only.py

@@ -0,0 +1,712 @@
+"""`litellm_content_filter` in `mode: logging_only`, driven through a real proxy.
+
+The provider is the only double, an owned wire peer. The proxy, its guardrail registry, Postgres and Redis run
+for real with two workers. Every check reads the spend row, which is written after the logging-only scans finish
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from hashlib import sha256
+from pathlib import Path
+from typing import Final
+
+import anthropic
+import openai
+import pytest
+import yaml
+from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, object_value
+from integration._support.database import read_rows
+from integration._support.mcp import EntryPoint, McpCaller, echo_tool, register_mcp, scripted_peer, tool_calls
+from integration._support.otlp_sink import AuditConfigWriter, SpanSinks, recorded_spans
+from integration._support.process import owned_proxy_process
+from integration._support.wire import Reply, Request, Wire, wire_server
+
+AWS_KEY: Final = "AKIA" + "SYNTHETIC0000000"
+GITHUB_TOKEN: Final = "ghp_" + "a1" * 18
+FORBIDDEN_WORD: Final = "synthetic-forbidden-word"
+LEAK_MARK: Final = "SYNTHETIC-LEAK-TOKEN"
+OUTAGE_MARK: Final = "SYNTHETIC-PROVIDER-OUTAGE"
+OBSERVE_BLOCK: Final = "cf-observe-block"
+OBSERVE_MASK: Final = "cf-observe-mask"
+ENFORCE_BLOCK: Final = "cf-enforce-block"
+ENFORCE_MASK: Final = "cf-enforce-mask"
+OBSERVE_CATEGORY: Final = "cf-observe-category"
+OBSERVE_COMPETITOR: Final = "cf-observe-competitor"
+SECRETS: Final = f"AWS_ACCESS_KEY_ID={AWS_KEY}\nGITHUB_TOKEN={GITHUB_TOKEN}"
+
+
+def _answer(marker: str) -> str:
+    return "synthetic answer " + marker
+
+
+def _provider_answer(body: bytes) -> str:
+    text: Final = body.decode()
+    start: Final = text.find("mark-")
+    assert start >= 0, text
+    answer: Final = _answer(text[start : start + 37])
+    return answer + " " + GITHUB_TOKEN if LEAK_MARK in text else answer
+
+
+def _sse(events: tuple[dict[str, object], ...]) -> tuple[bytes, ...]:
+    return tuple(("data: " + json.dumps(event) + "\n\n").encode() for event in events)
+
+
+def _chat_reply(answer: str, streaming: bool) -> Reply:
+    if not streaming:
+        return Reply(
+            body=json.dumps(
+                {
+                    "id": "chatcmpl-" + uuid.uuid4().hex,
+                    "object": "chat.completion",
+                    "created": 1,
+                    "model": "gpt-5.4-mini",
+                    "choices": [
+                        {"index": 0, "message": {"role": "assistant", "content": answer}, "finish_reason": "stop"}
+                    ],
+                    "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
+                }
+            ).encode()
+        )
+    identity: Final = "chatcmpl-" + uuid.uuid4().hex
+
+    def chunk(delta: dict[str, object], finish: str | None) -> dict[str, object]:
+        return {
+            "id": identity,
+            "object": "chat.completion.chunk",
+            "created": 1,
+            "model": "gpt-5.4-mini",
+            "choices": [{"index": 0, "delta": delta, "finish_reason": finish}],
+        }
+
+    events: Final = (chunk({"role": "assistant", "content": answer[:9]}, None), chunk({"content": answer[9:]}, "stop"))
+    return Reply(chunks=(*_sse(events), b"data: [DONE]\n\n"), content_type="text/event-stream")
+
+
+def _messages_reply(answer: str) -> Reply:
+    return Reply(
+        body=json.dumps(
+            {
+                "id": "msg_" + uuid.uuid4().hex,
+                "type": "message",
+                "role": "assistant",
+                "model": "claude-haiku-5-5",
+                "content": [{"type": "text", "text": answer}],
+                "stop_reason": "end_turn",
+                "stop_sequence": None,
+                "usage": {"input_tokens": 10, "output_tokens": 5},
+            }
+        ).encode()
+    )
+
+
+def _responses_reply(answer: str) -> Reply:
+    return Reply(
+        body=json.dumps(
+            {
+                "id": "resp_" + uuid.uuid4().hex,
+                "object": "response",
+                "created_at": 1,
+                "status": "completed",
+                "model": "gpt-5.4-mini",
+                "output": [
+                    {
+                        "type": "message",
+                        "id": "msgo_" + uuid.uuid4().hex,
+                        "status": "completed",
+                        "role": "assistant",
+                        "content": [{"type": "output_text", "text": answer, "annotations": []}],
+                    }
+                ],
+                "usage": {"input_tokens": 10, "output_tokens": 5, "total_tokens": 15},
+            }
+        ).encode()
+    )
+
+
+def _provider(request: Request) -> Reply:
+    if OUTAGE_MARK.encode() in request.body:
+        return Reply(status=503, body=b'{"error": {"message": "synthetic provider outage", "type": "server_error"}}')
+    answer: Final = _provider_answer(request.body)
+    if request.target.endswith("/v1/messages"):
+        return _messages_reply(answer)
+    if request.target.endswith("/v1/responses"):
+        return _responses_reply(answer)
+    assert request.target.endswith("/v1/chat/completions"), request.target
+    return _chat_reply(answer, bool(json.loads(request.body).get("stream")))
+
+
+def _content_filter(name: str, mode: str, action: str, *, default_on: bool) -> dict[str, object]:
+    return {
+        "guardrail_name": name,
+        "litellm_params": {
+            "guardrail": "litellm_content_filter",
+            "mode": mode,
+            "default_on": default_on,
+            "patterns": [
+                {"pattern_type": "regex", "pattern": "AKIA[0-9A-Z]{16}", "name": "aws_access_key", "action": action},
+                {"pattern_type": "regex", "pattern": "ghp_[A-Za-z0-9]{36}", "name": "github_pat", "action": action},
+            ],
+            "blocked_words": [{"keyword": FORBIDDEN_WORD, "action": action}],
+        },
+    }
+
+
+def _observe_only(name: str, **rules: object) -> dict[str, object]:
+    return {
+        "guardrail_name": name,
+        "litellm_params": {"guardrail": "litellm_content_filter", "mode": "logging_only", "default_on": False, **rules},
+    }
+
+
+def _rig_config(root: Path) -> Path:
+    config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
+    config["litellm_settings"]["cache"] = False
+    config["general_settings"]["store_prompts_in_spend_logs"] = True
+    config["guardrails"] = [
+        _content_filter(OBSERVE_BLOCK, "logging_only", "BLOCK", default_on=True),
+        _content_filter(OBSERVE_MASK, "logging_only", "MASK", default_on=True),
+        _content_filter(ENFORCE_BLOCK, "pre_call", "BLOCK", default_on=False),
+        _content_filter(ENFORCE_MASK, "pre_call", "MASK", default_on=False),
+        _observe_only(
+            OBSERVE_CATEGORY,
+            categories=[
+                {"category": "harmful_violence", "enabled": True, "action": "BLOCK", "severity_threshold": "medium"}
+            ],
+            blocked_words=[{"keyword": FORBIDDEN_WORD, "action": "BLOCK"}],
+        ),
+        _observe_only(
+            OBSERVE_COMPETITOR,
+            competitor_intent_config={
+                "brand_self": ["emirates"],
+                "competitors": ["qatar airways"],
+                "domain_words": ["airline", "flight"],
+                "policy": {"competitor_comparison": "refuse"},
+                "threshold_high": 0.5,
+            },
+        ),
+    ]
+    path: Final = root / "content-filter-logging-only.yaml"
+    path.write_text(yaml.safe_dump(config))
+    return path
+
+
+@dataclass(frozen=True, slots=True)
+class Rig:
+    proxy: Gateway
+    provider: Wire
+    chat_model: str
+    claude_model: str
+    responses_model: str
+    scenario: Scenario
+
+    def base(self) -> str:
+        return str(self.proxy.client.base_url).rstrip("/")
+
+    def provider_bodies(self, marker: str) -> tuple[str, ...]:
+        return tuple(request.body.decode() for request in self.provider.drain() if marker.encode() in request.body)
+
+    def entries(self, response_id: str, name: str) -> tuple[dict[str, object], ...]:
+        return _named_entries(
+            'SELECT request_id, status, metadata FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (response_id,), name
+        )
+
+
+def _named_entries(query: str, parameters: tuple[str, ...], name: str) -> tuple[dict[str, object], ...]:
+    rows: Final = eventually(lambda: read_rows(query, parameters), lambda values: len(values) >= 1, seconds=70)
+    assert len(rows) == 1, rows
+    recorded: Final = object_value(rows[0]["metadata"]).get("guardrail_information")
+    assert isinstance(recorded, list), rows[0]
+    return tuple(entry for entry in (object_value(item) for item in recorded) if entry.get("guardrail_name") == name)
+
+
+def _findings(entry: dict[str, object]) -> tuple[str, ...]:
+    response: Final = entry.get("guardrail_response")
+    assert isinstance(response, list), entry
+    return tuple(
+        str(detection.get("pattern_name") or detection.get("keyword") or detection.get("type"))
+        for detection in (object_value(item) for item in response)
+    )
+
+
+def _verdicts(entries: tuple[dict[str, object], ...]) -> list[tuple[object, object, tuple[str, ...]]]:
+    return [(entry["guardrail_mode"], entry["guardrail_status"], _findings(entry)) for entry in entries]
+
+
+@pytest.fixture(scope="module")
+def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Rig]:
+    root: Final = tmp_path_factory.mktemp("content-filter-logging-only")
+    with (
+        gateway_from_environment() as gateway,
+        wire_server(_provider) as provider,
+        owned_proxy_process(gateway, root, {}, config=_rig_config(root), workers=2) as owned,
+        owned.gateway.scenario() as scenario,
+    ):
+        chat: Final = scenario.model(
+            model="openai/gpt-5.4-mini", api_base=provider.url + "/v1", api_key="synthetic-openai-key"
+        )
+        claude: Final = scenario.model(
+            model="anthropic/claude-haiku-5-5", api_base=provider.url, api_key="synthetic-anthropic-key"
+        )
+        responses: Final = scenario.model(
+            model="openai/gpt-5.4-mini", api_base=provider.url + "/v1", api_key="synthetic-openai-key"
+        )
+        yield Rig(owned.gateway, provider, chat, claude, responses, scenario)
+
+
+def _marker() -> str:
+    return "mark-" + uuid.uuid4().hex
+
+
+def _sync_chat(rig: Rig, text: str) -> tuple[str, str]:
+    client: Final = openai.OpenAI(base_url=rig.base() + "/v1", api_key=rig.proxy.key, max_retries=0)
+    completion: Final = client.chat.completions.create(
+        model=rig.chat_model, messages=[{"role": "user", "content": text}]
+    )
+    return completion.id, completion.choices[0].message.content or ""
+
+
+async def _async_chat(rig: Rig, text: str) -> tuple[str, str]:
+    client: Final = openai.AsyncOpenAI(base_url=rig.base() + "/v1", api_key=rig.proxy.key, max_retries=0)
+    completion: Final = await client.chat.completions.create(
+        model=rig.chat_model, messages=[{"role": "user", "content": text}]
+    )
+    return completion.id, completion.choices[0].message.content or ""
+
+
+def _streamed_chat(rig: Rig, text: str) -> tuple[str, str]:
+    client: Final = openai.OpenAI(base_url=rig.base() + "/v1", api_key=rig.proxy.key, max_retries=0)
+    chunks: Final = tuple(
+        client.chat.completions.create(model=rig.chat_model, messages=[{"role": "user", "content": text}], stream=True)
+    )
+    return chunks[0].id, "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices)
+
+
+def _messages(rig: Rig, text: str) -> tuple[str, str]:
+    client: Final = anthropic.Anthropic(base_url=rig.base(), api_key=rig.proxy.key, max_retries=0)
+    message: Final = client.messages.create(
+        model=rig.claude_model, max_tokens=64, messages=[{"role": "user", "content": text}]
+    )
+    assert message.content[0].type == "text"
+    return message.id, message.content[0].text
+
+
+def _responses(rig: Rig, text: str) -> tuple[str, str]:
+    client: Final = openai.OpenAI(base_url=rig.base() + "/v1", api_key=rig.proxy.key, max_retries=0)
+    result: Final = client.responses.create(model=rig.responses_model, input=text)
+    return result.id, result.output_text
+
+
+SYNC_CALLS: Final = {"chat": _sync_chat, "chat_stream": _streamed_chat, "messages": _messages, "responses": _responses}
+EVERY_FINDING: Final = ("aws_access_key", "github_pat", FORBIDDEN_WORD)
+
+
+@pytest.mark.parametrize("endpoint", list(SYNC_CALLS))
+def test_logging_only_block_records_every_match_without_touching_the_call(rig: Rig, endpoint: str) -> None:
+    marker: Final = _marker()
+    prompt: Final = f"{marker} what did the file contain?\n{SECRETS}\n{FORBIDDEN_WORD}"
+    response_id, answer = SYNC_CALLS[endpoint](rig, prompt)
+    assert answer == _answer(marker)
+    sent: Final = rig.provider_bodies(marker)
+    assert len(sent) == 1 and AWS_KEY in sent[0] and GITHUB_TOKEN in sent[0] and FORBIDDEN_WORD in sent[0], sent
+
+    assert _verdicts(rig.entries(response_id, OBSERVE_BLOCK)) == [
+        ("logging_only", "guardrail_flagged", EVERY_FINDING),
+        ("logging_only", "success", ()),
+    ]
+
+
+@pytest.mark.asyncio
+async def test_logging_only_block_records_every_match_for_an_async_client(rig: Rig) -> None:
+    marker: Final = _marker()
+    response_id, answer = await _async_chat(rig, f"{marker} {SECRETS}")
+    assert answer == _answer(marker)
+
+    assert _verdicts(rig.entries(response_id, OBSERVE_BLOCK)) == [
+        ("logging_only", "guardrail_flagged", ("aws_access_key", "github_pat")),
+        ("logging_only", "success", ()),
+    ]
+
+
+def test_logging_only_block_still_scans_the_response_after_a_request_match(rig: Rig) -> None:
+    marker: Final = _marker()
+    response_id, answer = _sync_chat(rig, f"{marker} {LEAK_MARK} key {AWS_KEY}")
+    assert answer == _answer(marker) + " " + GITHUB_TOKEN
+
+    assert _verdicts(rig.entries(response_id, OBSERVE_BLOCK)) == [
+        ("logging_only", "guardrail_flagged", ("aws_access_key",)),
+        ("logging_only", "guardrail_flagged", ("github_pat",)),
+    ]
+
+
+def _tool_call_history(text: str, arguments: str) -> list[dict[str, object]]:
+    return [
+        {"role": "user", "content": text},
+        {
+            "role": "assistant",
+            "content": None,
+            "tool_calls": [
+                {"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": arguments}}
+            ],
+        },
+        {"role": "tool", "tool_call_id": "call_1", "content": "done"},
+    ]
+
+
+@pytest.mark.parametrize(
+    ("arguments", "request_findings"),
+    (
+        pytest.param(json.dumps({AWS_KEY: 1}), ("aws_access_key",), id="masked-argument-key"),
+        pytest.param("[" * 150 + "1" + "]" * 150, (), id="too-deep"),
+    ),
+)
+def test_logging_only_flags_tool_arguments_an_enforcing_mode_rejects_and_keeps_scanning(
+    rig: Rig, arguments: str, request_findings: tuple[str, ...]
+) -> None:
+    marker: Final = _marker()
+    messages: Final = _tool_call_history(f"{marker} {LEAK_MARK}", arguments)
+    headers: Final = {"Authorization": f"Bearer {rig.proxy.key}"}
+    enforced: Final = rig.proxy.client.post(
+        "/v1/chat/completions",
+        json={"model": rig.chat_model, "messages": messages, "guardrails": [ENFORCE_MASK]},
+        headers=headers,
+    )
+    assert enforced.status_code == 400, enforced.text
+    assert rig.provider_bodies(marker) == ()
+
+    observed: Final = rig.proxy.client.post(
+        "/v1/chat/completions", json={"model": rig.chat_model, "messages": messages}, headers=headers
+    )
+    assert observed.status_code == 200, observed.text
+    completion: Final = object_value(observed.json())
+    assert len(rig.provider_bodies(marker)) == 1
+
+    assert _verdicts(rig.entries(str(completion["id"]), OBSERVE_MASK)) == [
+        ("logging_only", "guardrail_flagged", request_findings),
+        ("logging_only", "success", ("github_pat",)),
+    ]
+
+
+@pytest.mark.parametrize("entry", ("mcp", "rest"))
+def test_logging_only_block_records_every_match_on_an_mcp_tool_call(rig: Rig, entry: EntryPoint) -> None:
+    alias: Final = "cfmcp" + uuid.uuid4().hex[:8]
+    arguments: Final = {"note": f"{_marker()} {SECRETS}"}
+    with scripted_peer(echo_tool("echo")) as peer:
+        server_id: Final = register_mcp(rig.scenario, peer, alias)
+        key: Final = rig.scenario.key(object_permission={"mcp_servers": [server_id]})
+        outcome: Final = McpCaller(rig.proxy, key, entry, alias).call(f"{alias}-echo", arguments, server_id=server_id)
+        assert (outcome.error, outcome.text) == (None, json.dumps(arguments, sort_keys=True)), outcome.raw
+        assert [call["body"]["params"]["arguments"] for call in tool_calls(peer.drain())] == [arguments]
+
+    entries: Final = _named_entries(
+        'SELECT metadata FROM "LiteLLM_SpendLogs" WHERE api_key = %s AND call_type = %s',
+        (sha256(key.encode()).hexdigest(), "call_mcp_tool"),
+        OBSERVE_BLOCK,
+    )
+    assert _verdicts(entries) == [("logging_only", "guardrail_flagged", ("aws_access_key", "github_pat"))], entries
+
+
+@pytest.mark.parametrize(
+    ("guardrail", "prompt", "findings"),
+    (
+        pytest.param(
+            OBSERVE_CATEGORY,
+            f"how to kill someone {FORBIDDEN_WORD}",
+            ("how to kill someone", FORBIDDEN_WORD),
+            id="category-keyword",
+        ),
+        pytest.param(
+            OBSERVE_COMPETITOR, "Is Qatar Airways better than Emirates?", ("competitor_intent",), id="competitor-intent"
+        ),
+    ),
+)
+def test_logging_only_block_flags_category_and_competitor_matches_without_blocking(
+    rig: Rig, guardrail: str, prompt: str, findings: tuple[str, ...]
+) -> None:
+    marker: Final = _marker()
+    response: Final = rig.proxy.client.post(
+        "/v1/chat/completions",
+        json={
+            "model": rig.chat_model,
+            "messages": [{"role": "user", "content": f"{marker} {prompt}"}],
+            "guardrails": [guardrail],
+        },
+        headers={"Authorization": f"Bearer {rig.proxy.key}"},
+    )
+    assert response.status_code == 200, response.text
+    completion: Final = object_value(response.json())
+    assert len(rig.provider_bodies(marker)) == 1
+
+    assert _verdicts(rig.entries(str(completion["id"]), guardrail)) == [
+        ("logging_only", "guardrail_flagged", findings),
+        ("logging_only", "success", ()),
+    ]
+
+
+def test_logging_only_mask_records_every_match(rig: Rig) -> None:
+    marker: Final = _marker()
+    response_id, answer = _sync_chat(rig, f"{marker} {SECRETS}\n{FORBIDDEN_WORD}")
+    assert answer == _answer(marker)
+
+    entries: Final = rig.entries(response_id, OBSERVE_MASK)
+    assert _verdicts(entries) == [
+        ("logging_only", "success", EVERY_FINDING),
+        ("logging_only", "success", ()),
+    ]
+    assert entries[0]["masked_entity_count"] == {"aws_access_key": 1, "github_pat": 1, "blocked_word": 1}, entries
+
+
+def test_logging_only_records_nothing_for_a_clean_call(rig: Rig) -> None:
+    marker: Final = _marker()
+    response_id, answer = _sync_chat(rig, f"{marker} nothing secret here")
+    assert answer == _answer(marker)
+
+    assert _verdicts(rig.entries(response_id, OBSERVE_BLOCK)) == [
+        ("logging_only", "success", ()),
+        ("logging_only", "success", ()),
+    ]
+
+
+def test_pre_call_block_still_rejects_the_request(rig: Rig) -> None:
+    marker: Final = _marker()
+    response: Final = rig.proxy.client.post(
+        "/v1/chat/completions",
+        json={
+            "model": rig.chat_model,
+            "messages": [{"role": "user", "content": f"{marker} {SECRETS}"}],
+            "guardrails": [ENFORCE_BLOCK],
+        },
+        headers={"Authorization": f"Bearer {rig.proxy.key}"},
+    )
+    assert response.status_code == 400, response.text
+    assert "aws_access_key" in response.text
+    assert rig.provider_bodies(marker) == ()
+
+
+def test_apply_guardrail_endpoint_still_blocks_with_a_logging_only_content_filter(rig: Rig) -> None:
+    response: Final = rig.proxy.client.post(
+        "/guardrails/apply_guardrail",
+        json={"guardrail_name": OBSERVE_BLOCK, "text": f"key {AWS_KEY}"},
+        headers={"Authorization": f"Bearer {rig.proxy.key}"},
+    )
+    assert response.status_code == 400, response.text
+    assert "aws_access_key" in response.text
+
+
+def _advertised_modes(rig: Rig) -> dict[str, object]:
+    response: Final = rig.proxy.client.get(
+        "/guardrails/ui/add_guardrail_settings", headers={"Authorization": f"Bearer {rig.proxy.key}"}
+    )
+    assert response.status_code == 200, response.text
+    return object_value(response.json()["supported_modes_by_provider"])
+
+
+@pytest.mark.parametrize("provider", ["litellm_content_filter", "generic_guardrail_api", "bedrock", "akto"])
+def test_dashboard_offers_logging_only_where_the_config_accepts_it(rig: Rig, provider: str) -> None:
+    modes: Final = _advertised_modes(rig)
+    offered: Final = modes[provider]
+    assert isinstance(offered, list) and "logging_only" in offered, modes
+    assert offered.count("logging_only") == 1, offered
+
+
+@pytest.mark.parametrize("provider", ["azure/prompt_shield", "lakera_v2"])
+def test_dashboard_does_not_offer_logging_only_where_the_config_rejects_it(rig: Rig, provider: str) -> None:
+    offered: Final = _advertised_modes(rig)[provider]
+    assert isinstance(offered, list) and "logging_only" not in offered, offered
+
+
+def test_a_logging_only_content_filter_created_through_the_api_records_every_match(rig: Rig) -> None:
+    name: Final = "cf-observe-api-" + uuid.uuid4().hex[:8]
+    created: Final = rig.proxy.client.post(
+        "/guardrails",
+        json={"guardrail": {"guardrail_name": name, **_content_filter(name, "logging_only", "BLOCK", default_on=True)}},
+        headers={"Authorization": f"Bearer {rig.proxy.key}"},
+    )
+    assert created.status_code == 200, created.text
+    guardrail_id: Final = str(object_value(created.json())["guardrail_id"])
+    try:
+        marker: Final = _marker()
+        response_id: Final = eventually(
+            lambda: _sync_chat(rig, f"{marker} {SECRETS}")[0],
+            lambda identity: bool(rig.entries(identity, name)),
+            seconds=70,
+        )
+        assert _verdicts(rig.entries(response_id, name))[0] == (
+            "logging_only",
+            "guardrail_flagged",
+            ("aws_access_key", "github_pat"),
+        )
+    finally:
+        rig.proxy.client.delete(f"/guardrails/{guardrail_id}", headers={"Authorization": f"Bearer {rig.proxy.key}"})
+
+
+def test_guardrails_monitor_counts_observe_only_matches_as_flagged_not_blocked(rig: Rig) -> None:
+    name: Final = "cf-observe-usage-" + uuid.uuid4().hex[:8]
+    headers: Final = {"Authorization": f"Bearer {rig.proxy.key}"}
+    created: Final = rig.proxy.client.post(
+        "/guardrails",
+        json={"guardrail": {"guardrail_name": name, **_content_filter(name, "logging_only", "BLOCK", default_on=True)}},
+        headers=headers,
+    )
+    assert created.status_code == 200, created.text
+    guardrail_id: Final = str(object_value(created.json())["guardrail_id"])
+    try:
+        eventually(
+            lambda: _sync_chat(rig, f"{_marker()} {SECRETS}")[0],
+            lambda identity: bool(rig.entries(identity, name)),
+            seconds=70,
+        )
+        today: Final = datetime.now(timezone.utc).date().isoformat()
+        detail: Final = eventually(
+            lambda: object_value(
+                rig.proxy.client.get(
+                    f"/guardrails/usage/detail/{guardrail_id}",
+                    params={"start_date": today, "end_date": today},
+                    headers=headers,
+                ).json()
+            ),
+            lambda body: int(str(body.get("requestsEvaluated", 0))) >= 1,
+            seconds=60,
+        )
+        assert (detail["failRate"], detail["status"]) == (0.0, "healthy"), detail
+    finally:
+        rig.proxy.client.delete(f"/guardrails/{guardrail_id}", headers=headers)
+
+
+@pytest.mark.parametrize(
+    ("environment", "name_key", "status_key"),
+    (
+        pytest.param({}, "guardrail_name", "guardrail_status", id="otel-v1"),
+        pytest.param({"LITELLM_OTEL_V2": "1"}, "litellm.guardrail.name", "litellm.guardrail.status", id="otel-v2"),
+    ),
+)
+def test_otel_guardrail_spans_carry_the_flagged_status_without_an_error(
+    gateway: Gateway,
+    audit_sinks: SpanSinks,
+    otel_audit_config: AuditConfigWriter,
+    tmp_path: Path,
+    environment: dict[str, str],
+    name_key: str,
+    status_key: str,
+) -> None:
+    config_path: Final = otel_audit_config(tmp_path, {"cache": False})
+    config: Final = yaml.safe_load(config_path.read_text())
+    config["guardrails"] = [_content_filter(OBSERVE_BLOCK, "logging_only", "BLOCK", default_on=True)]
+    config_path.write_text(yaml.safe_dump(config))
+    overrides: Final = {"OTEL_EXPORTER": "http/json", "OTEL_ENDPOINT": audit_sinks.operator, **environment}
+    with (
+        wire_server(_provider) as provider,
+        owned_proxy_process(gateway, tmp_path, overrides, config=config_path, workers=2) as owned,
+        owned.gateway.scenario() as scenario,
+    ):
+        model: Final = scenario.model(
+            model="openai/gpt-5.4-mini", api_base=provider.url + "/v1", api_key="synthetic-openai-key"
+        )
+        start, _ = recorded_spans(audit_sinks.operator)
+        marker: Final = _marker()
+        client: Final = openai.OpenAI(
+            base_url=str(owned.gateway.client.base_url).rstrip("/") + "/v1", api_key=owned.gateway.key, max_retries=0
+        )
+        completion: Final = client.chat.completions.create(
+            model=model, messages=[{"role": "user", "content": f"{marker} {SECRETS}"}]
+        )
+        assert completion.choices[0].message.content == _answer(marker)
+
+        spans: Final = eventually(
+            lambda: tuple(
+                span
+                for span in recorded_spans(audit_sinks.operator, start)[1]
+                if span["attributes"].get(name_key) == OBSERVE_BLOCK
+            ),
+            lambda found: len(found) >= 2,
+            seconds=60,
+            return_last_on_timeout=True,
+        )
+        assert sorted(
+            (str(span["attributes"].get(status_key)), "error.type" in span["attributes"]) for span in spans
+        ) == [("guardrail_flagged", False), ("success", False)], spans
+
+
+def _burst_call(rig: Rig, index: int) -> tuple[str, int, str, str]:
+    marker: Final = _marker()
+    if index % 4 == 0:
+        enforced: Final = rig.proxy.client.post(
+            "/v1/chat/completions",
+            json={
+                "model": rig.chat_model,
+                "messages": [{"role": "user", "content": f"{marker} {SECRETS}"}],
+                "guardrails": [ENFORCE_BLOCK],
+            },
+            headers={"Authorization": f"Bearer {rig.proxy.key}"},
+        )
+        return "enforce", enforced.status_code, marker, ""
+    if index % 4 == 1:
+        failed: Final = rig.proxy.client.post(
+            "/v1/chat/completions",
+            json={
+                "model": rig.chat_model,
+                "messages": [{"role": "user", "content": f"{marker} {OUTAGE_MARK} {SECRETS}"}],
+            },
+            headers={"Authorization": f"Bearer {rig.proxy.key}"},
+        )
+        return "outage", failed.status_code, marker, failed.headers.get("x-litellm-call-id", "")
+    endpoint: Final = list(SYNC_CALLS)[index % len(SYNC_CALLS)]
+    response_id, answer = SYNC_CALLS[endpoint](rig, f"{marker} {SECRETS}\n{FORBIDDEN_WORD}")
+    assert answer == _answer(marker), (endpoint, answer)
+    return "observe:" + endpoint, 200, marker, response_id
+
+
+def _failure_row(call_id: str) -> tuple[object, tuple[dict[str, object], ...]]:
+    rows: Final = eventually(
+        lambda: read_rows('SELECT status, metadata FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (call_id,)),
+        lambda values: len(values) >= 1,
+        seconds=70,
+    )
+    assert len(rows) == 1, rows
+    recorded: Final = object_value(rows[0]["metadata"]).get("guardrail_information") or []
+    assert isinstance(recorded, list), rows[0]
+    return rows[0]["status"], tuple(
+        entry for entry in (object_value(item) for item in recorded) if entry.get("guardrail_name") == OBSERVE_BLOCK
+    )
+
+
+def test_concurrent_burst_with_a_provider_outage_records_every_observed_call_once(rig: Rig) -> None:
+    def call(index: int) -> tuple[str, int, str, str]:
+        return _burst_call(rig, index)
+
+    with ThreadPoolExecutor(max_workers=12) as pool:
+        results: Final = tuple(pool.map(call, range(32)))
+
+    assert sorted({(kind.split(":")[0], status) for kind, status, _, _ in results}) == [
+        ("enforce", 400),
+        ("observe", 200),
+        ("outage", 503),
+    ], results
+    sent: Final = rig.provider_bodies("mark-")
+    for kind, _, marker, _ in results:
+        if kind == "enforce":
+            assert not any(marker in body for body in sent), marker
+    outages: Final = tuple(call_id for kind, _, _, call_id in results if kind == "outage")
+    assert len(outages) == 8 and all(outages), results
+    assert [_failure_row(call_id) for call_id in outages] == [("failure", ())] * 8
+    observed: Final = tuple((kind, response_id) for kind, _, _, response_id in results if kind.startswith("observe"))
+    assert len(observed) == 16
+    for kind, response_id in observed:
+        assert _verdicts(rig.entries(response_id, OBSERVE_BLOCK)) == [
+            ("logging_only", "guardrail_flagged", EVERY_FINDING),
+            ("logging_only", "success", ()),
+        ], (kind, response_id)
+
+    marker: Final = _marker()
+    response_id, answer = _sync_chat(rig, f"{marker} {SECRETS}")
+    assert answer == _answer(marker)
+    assert _verdicts(rig.entries(response_id, OBSERVE_BLOCK))[0] == (
+        "logging_only",
+        "guardrail_flagged",
+        ("aws_access_key", "github_pat"),
+    )

tests/integration/observability/test_logging_only_scope_config.py

@@ -179,16 +179,16 @@ def test_native_content_filter_scope_logs_without_blocking(
                     (
                         identity,
                         "logging_only",
-                        "guardrail_intervened" if direction == blocked_side else "success",
+                        "guardrail_flagged" if direction == blocked_side else "success",
                     )
                     for direction in expected_directions
                 ), entries
-                intervened_entries: Final = tuple(
-                    entry for entry in entries if entry["guardrail_status"] == "guardrail_intervened"
+                flagged_entries: Final = tuple(
+                    entry for entry in entries if entry["guardrail_status"] == "guardrail_flagged"
                 )
-                assert len(intervened_entries) == 1, entries
-                assert intervened_entries[0]["guardrail_response"] is not None, entries
-                assert intervened_entries[0]["guardrail_response"] == "REDACTED_BY_LITELM", entries
+                assert len(flagged_entries) == 1, entries
+                assert flagged_entries[0]["guardrail_response"] is not None, entries
+                assert flagged_entries[0]["guardrail_response"] == "REDACTED_BY_LITELM", entries
     finally:
         delete_scenario(upstream_handle)
 

tests/integration/observability/test_otel_excluded_services_matrix.py

@@ -358,6 +358,10 @@ def _has_root(spans: tuple[Span, ...]) -> bool:
     return any(span["kind"] == SERVER for span in spans)
 
 
+def _has_model_span(spans: tuple[Span, ...]) -> bool:
+    return any("gen_ai.operation.name" in span["attributes"] for span in spans)
+
+
 def _trace_of_call(sink: str, call_id: str, since: int) -> tuple[Span, ...]:
     _, spans = recorded_spans(sink, since)
     traces: Final = {span["trace_id"] for span in spans if span["attributes"].get("litellm.call_id") == call_id}
@@ -367,7 +371,7 @@ def _trace_of_call(sink: str, call_id: str, since: int) -> tuple[Span, ...]:
 def _operator_trace(rig: Rig, sent: Sent, cursors: Cursors) -> tuple[Span, ...]:
     trace: Final = eventually(
         lambda: _trace_of_call(rig.sinks.operator, sent.call_id, cursors.operator),
-        lambda spans: _has_root(spans) and "redis" in _db_systems(spans),
+        lambda spans: _has_root(spans) and "redis" in _db_systems(spans) and _has_model_span(spans),
         seconds=40,
     )
     assert len({span["trace_id"] for span in trace}) == 1, _names(trace)
@@ -405,7 +409,7 @@ def _assert_tenant_mirrors(rig: Rig, operator: tuple[Span, ...], cursors: Cursor
 
 def _assert_withheld(rig: Rig, sent: Sent, cursors: Cursors) -> tuple[Span, ...]:
     tenant: Final = _assert_tenant_mirrors(rig, _operator_trace(rig, sent, cursors), cursors)
-    assert any("gen_ai.operation.name" in span["attributes"] for span in tenant), _names(tenant)
+    assert _has_model_span(tenant), _names(tenant)
     return tenant
 
 
@@ -719,7 +723,7 @@ def _assert_tenant_kept(
         return (
             sum(1 for span in spans if span["kind"] == SERVER) == 1
             and "redis" in _db_systems(spans)
-            and (not needs_model_span or any("gen_ai.operation.name" in span["attributes"] for span in spans))
+            and (not needs_model_span or _has_model_span(spans))
         )
 
     tenant: Final = eventually(
@@ -730,7 +734,7 @@ def _assert_tenant_kept(
     )
     assert sum(1 for span in tenant if span["kind"] == SERVER) == 1, _names(tenant)
     assert "redis" in _db_systems(tenant), f"redis spans missing at the tenant: {_names(tenant)}"
-    assert not needs_model_span or any("gen_ai.operation.name" in span["attributes"] for span in tenant), _names(tenant)
+    assert not needs_model_span or _has_model_span(tenant), _names(tenant)
     return tenant
 
 

tests/integration/providers/test_anthropic_empty_user_turn_wire.py

@@ -0,0 +1,216 @@
+import json
+from collections.abc import Iterator, Mapping, Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from types import MappingProxyType
+from typing import Final
+
+import pytest
+import yaml
+from integration._support.client import Gateway, gateway_from_environment, object_value
+from integration._support.process import graceful_stop_seconds, owned_proxy
+from integration._support.wire import Reply, Request, Wire, wire_server
+from integration.providers._cache_control_marks_support import (
+    anthropic_deployment,
+    anthropic_peer,
+    final_text,
+    new_marker,
+)
+from pydantic import JsonValue, TypeAdapter
+
+pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
+
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_MODEL: Final = "empty-user-turn"
+_PLACEHOLDER: Final = "[System: Empty message content sanitised to satisfy protocol]"
+_REPLY: Final = "Noted."
+_EMPTY_ROWS: Final[Mapping[str, Mapping[str, JsonValue]]] = MappingProxyType(
+    {
+        "list": MappingProxyType({"role": "user", "content": []}),
+        "null": MappingProxyType({"role": "user", "content": None}),
+        "missing": MappingProxyType({"role": "user"}),
+        "empty-string": MappingProxyType({"role": "user", "content": ""}),
+    }
+)
+_CHAT_EMPTY_KINDS: Final = ("list", "null", "missing")
+
+Turn = tuple[str, tuple[str, ...]]
+
+
+@dataclass(frozen=True, slots=True)
+class _Rig:
+    gateway: Gateway
+    provider: Wire
+
+
+def _empty_row(kind: str) -> dict[str, JsonValue]:
+    return dict(_EMPTY_ROWS[kind])
+
+
+def _guardrail_peer(request: Request) -> Reply:
+    kind: Final = request.target.split("/")[1]
+    rows: Final = _JSON_OBJECT.validate_json(request.body)["structured_messages"]
+    assert isinstance(rows, list), rows
+    verdict: Final = {"action": "GUARDRAIL_INTERVENED", "structured_messages": [*rows, _empty_row(kind)]}
+    return Reply(body=json.dumps(verdict).encode())
+
+
+def _guardrail(kind: str, api_base: str) -> dict[str, JsonValue]:
+    return {
+        "guardrail_name": f"append-{kind}",
+        "litellm_params": {
+            "guardrail": "generic_guardrail_api",
+            "mode": "pre_call",
+            "default_on": False,
+            "api_base": f"{api_base}/{kind}",
+            "api_key": "synthetic-guardrail-key",
+        },
+    }
+
+
+def _config(directory: Path, provider_url: str, guardrail_url: str) -> Path:
+    stock: Final = _JSON_OBJECT.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()))
+    config: Final[dict[str, JsonValue]] = {
+        **stock,
+        "model_list": [anthropic_deployment(_MODEL, provider_url)],
+        "router_settings": {**object_value(stock["router_settings"]), "num_retries": 0},
+        "guardrails": [_guardrail(kind, guardrail_url) for kind in _EMPTY_ROWS],
+    }
+    path: Final = directory / "empty-user-turn.yaml"
+    path.write_text(yaml.safe_dump(config))
+    return path
+
+
+@pytest.fixture(scope="module")
+def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]:
+    directory: Final = tmp_path_factory.mktemp("empty-user-turn")
+    with (
+        wire_server(anthropic_peer) as provider,
+        wire_server(_guardrail_peer) as guardrail,
+        gateway_from_environment() as gateway,
+    ):
+        config: Final = _config(directory, provider.url, guardrail.url)
+        with owned_proxy(gateway, directory, {}, config=config, workers=2) as owned:
+            yield _Rig(owned, provider)
+
+
+def _block(block: JsonValue) -> str:
+    part: Final = object_value(block)
+    return str(part["text"]) if part.get("type") == "text" else str(part.get("type"))
+
+
+def _turn(message: JsonValue) -> Turn:
+    row: Final = object_value(message)
+    content: Final = row["content"]
+    if isinstance(content, str):
+        return (str(row["role"]), (content,))
+    assert isinstance(content, list), row
+    return (str(row["role"]), tuple(_block(block) for block in content))
+
+
+def _send(rig: _Rig, path: str, body: Mapping[str, JsonValue]) -> None:
+    response: Final = rig.gateway.client.post(
+        path, json=dict(body), headers={"Authorization": f"Bearer {rig.gateway.key}"}
+    )
+    assert response.status_code == 200, response.text
+
+
+def _provider_turns(rig: _Rig, marker: str) -> tuple[Turn, ...]:
+    needle: Final = marker.encode()
+    requests: Final = [request for request in rig.provider.drain() if needle in request.body]
+    assert len(requests) == 1, requests
+    messages: Final = _JSON_OBJECT.validate_json(requests[0].body)["messages"]
+    assert isinstance(messages, list), messages
+    return tuple(_turn(message) for message in messages)
+
+
+def _ask_row(ask: str) -> dict[str, JsonValue]:
+    return {"role": "user", "content": ask}
+
+
+def _chat(messages: Sequence[JsonValue]) -> dict[str, JsonValue]:
+    return {"model": _MODEL, "max_tokens": 64, "messages": list(messages)}
+
+
+@pytest.mark.parametrize("kind", _CHAT_EMPTY_KINDS)
+def test_chat_trailing_empty_user_message_keeps_a_user_turn(rig: _Rig, kind: str) -> None:
+    marker: Final = new_marker()
+    ask: Final = final_text(marker)
+    _send(
+        rig,
+        "/v1/chat/completions",
+        _chat([_ask_row(ask), {"role": "assistant", "content": _REPLY}, _empty_row(kind)]),
+    )
+    assert _provider_turns(rig, marker) == (("user", (ask,)), ("assistant", (_REPLY,)), ("user", (_PLACEHOLDER,)))
+
+
+@pytest.mark.parametrize("kind", _CHAT_EMPTY_KINDS)
+def test_chat_empty_user_message_beside_text_sends_only_the_text(rig: _Rig, kind: str) -> None:
+    marker: Final = new_marker()
+    ask: Final = final_text(marker)
+    _send(rig, "/v1/chat/completions", _chat([_ask_row(ask), _empty_row(kind)]))
+    assert _provider_turns(rig, marker) == (("user", (ask,)),)
+
+
+def _system_then_empty_turns(rig: _Rig, kind: str) -> tuple[Turn, ...]:
+    marker: Final = new_marker()
+    ask: Final = final_text(marker)
+    _send(
+        rig,
+        "/v1/chat/completions",
+        _chat(
+            [
+                _ask_row(ask),
+                {"role": "assistant", "content": _REPLY},
+                {"role": "system", "content": "Answer in French."},
+                _empty_row(kind),
+            ]
+        ),
+    )
+    turns: Final = _provider_turns(rig, marker)
+    assert turns[0] == ("user", (ask,)), turns
+    return turns[1:]
+
+
+@pytest.mark.parametrize("kind", _CHAT_EMPTY_KINDS)
+def test_chat_system_message_before_an_empty_user_message_is_placed_like_an_empty_string(rig: _Rig, kind: str) -> None:
+    assert _system_then_empty_turns(rig, kind) == _system_then_empty_turns(rig, "empty-string")
+
+
+def test_responses_trailing_empty_input_message_keeps_a_user_turn(rig: _Rig) -> None:
+    marker: Final = new_marker()
+    ask: Final = final_text(marker)
+    body: Final[dict[str, JsonValue]] = {
+        "model": _MODEL,
+        "max_output_tokens": 64,
+        "input": [
+            _ask_row(ask),
+            {"role": "assistant", "content": _REPLY},
+            {"role": "user", "content": []},
+        ],
+    }
+    _send(rig, "/v1/responses", body)
+    assert _provider_turns(rig, marker) == (("user", (ask,)), ("assistant", (_REPLY,)), ("user", (_PLACEHOLDER,)))
+
+
+@pytest.mark.parametrize("kind", tuple(_EMPTY_ROWS))
+def test_messages_guardrail_empty_user_row_is_sent_like_an_empty_string(rig: _Rig, kind: str) -> None:
+    marker: Final = new_marker()
+    ask: Final = final_text(marker)
+    body: Final[dict[str, JsonValue]] = {
+        "model": _MODEL,
+        "max_tokens": 64,
+        "messages": [_ask_row(ask)],
+        "guardrails": [f"append-{kind}"],
+    }
+    _send(rig, "/v1/messages", body)
+    assert _provider_turns(rig, marker) == (("user", (ask,)), ("user", (_PLACEHOLDER,)))
+
+
+def test_a_long_run_of_empty_user_messages_is_sent_as_one_text_turn(rig: _Rig) -> None:
+    marker: Final = new_marker()
+    ask: Final = final_text(marker)
+    _send(
+        rig, "/v1/chat/completions", _chat([*({"role": "user", "content": []} for _ in range(10_000)), _ask_row(ask)])
+    )
+    assert _provider_turns(rig, marker) == (("user", (ask,)),)

tests/integration/providers/test_count_tokens_error_shapes_wire.py

@@ -0,0 +1,205 @@
+import json
+from collections.abc import Iterator
+from typing import Final
+
+import anthropic
+import openai
+import pytest
+from integration._support.client import Gateway, gateway_from_environment
+from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
+from integration._support.wire import Reply, Request, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
+
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_MESSAGES: Final[list[JsonValue]] = [{"role": "user", "content": "Count this synthetic request."}]
+_PROVIDER_KEY_VARIABLES: Final = (
+    "ANTHROPIC_API_KEY",
+    "ANTHROPIC_AUTH_TOKEN",
+    "AZURE_AI_API_KEY",
+    "AZURE_API_KEY",
+    "AZURE_OPENAI_API_KEY",
+    "AZURE_AD_TOKEN",
+    "AZURE_CLIENT_ID",
+    "AZURE_CLIENT_SECRET",
+    "AZURE_TENANT_ID",
+    "OPENAI_API_KEY",
+)
+
+
+@pytest.fixture(scope="module")
+def keyless_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]:
+    with (
+        gateway_from_environment() as gateway,
+        owned_proxy_process(
+            gateway,
+            tmp_path_factory.mktemp("count-tokens-error-shapes"),
+            {},
+            remove_environment=_PROVIDER_KEY_VARIABLES,
+            workers=1,
+        ) as owned,
+    ):
+        yield owned
+
+
+@pytest.fixture(scope="module")
+def azure_environment_key_proxy(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]:
+    with (
+        gateway_from_environment() as gateway,
+        owned_proxy_process(
+            gateway,
+            tmp_path_factory.mktemp("count-tokens-azure-environment-key"),
+            {"AZURE_API_KEY": "synthetic-azure-env-key"},
+            remove_environment=tuple(name for name in _PROVIDER_KEY_VARIABLES if name != "AZURE_API_KEY"),
+            workers=1,
+        ) as owned,
+    ):
+        yield owned
+
+
+def _anthropic(gateway: Gateway) -> anthropic.Anthropic:
+    return anthropic.Anthropic(
+        api_key=gateway.key, base_url=str(gateway.client.base_url), max_retries=0, http_client=gateway.client
+    )
+
+
+def _never_called(request: Request) -> Reply:
+    raise AssertionError(f"count reached the provider without a key: {request.target}")
+
+
+@pytest.mark.parametrize(
+    ("deployment", "missing"),
+    (
+        ({"model": "anthropic/claude-sonnet-4-6"}, "No Anthropic API key is set for model "),
+        ({"model": "azure_ai/claude-sonnet-4-6"}, "No Azure AI API key or Entra ID credential is set for model "),
+    ),
+    ids=("anthropic", "azure_ai_anthropic"),
+)
+def test_anthropic_count_without_a_provider_credential_is_an_anthropic_401(
+    keyless_proxy: OwnedProxy, deployment: dict[str, JsonValue], missing: str
+) -> None:
+    gateway: Final = keyless_proxy.gateway
+    with wire_server(_never_called) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(**deployment, api_base=wire.url, api_key=None, num_retries=0)
+        with pytest.raises(anthropic.AuthenticationError) as raised:
+            _anthropic(gateway).messages.count_tokens(model=model, messages=_MESSAGES)
+        assert raised.value.body == {
+            "type": "error",
+            "error": {"type": "authentication_error", "message": raised.value.body["error"]["message"]},
+        }
+        assert missing in raised.value.message, raised.value.message
+        assert wire.drain() == ()
+
+
+def test_azure_ai_anthropic_count_sends_the_azure_environment_key_as_x_api_key(
+    azure_environment_key_proxy: OwnedProxy,
+) -> None:
+    gateway: Final = azure_environment_key_proxy.gateway
+
+    def count(request: Request) -> Reply:
+        assert request.headers.get("x-api-key") == "synthetic-azure-env-key", request.headers
+        return Reply(body=json.dumps({"input_tokens": 58}).encode())
+
+    with wire_server(count) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="azure_ai/claude-sonnet-4-6",
+            api_base=wire.url,
+            api_key=None,
+            num_retries=0,
+        )
+        result: Final = _anthropic(gateway).messages.count_tokens(model=model, messages=_MESSAGES)
+        assert result.input_tokens == 58
+
+
+def test_azure_ai_anthropic_count_authenticates_with_an_entra_token(keyless_proxy: OwnedProxy) -> None:
+    gateway: Final = keyless_proxy.gateway
+
+    def count(request: Request) -> Reply:
+        assert request.headers.get("authorization") == "Bearer synthetic-entra-token", request.headers
+        assert "x-api-key" not in request.headers, request.headers
+        return Reply(body=json.dumps({"input_tokens": 61}).encode())
+
+    with wire_server(count) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="azure_ai/claude-sonnet-4-6",
+            api_base=wire.url,
+            api_key=None,
+            azure_ad_token="synthetic-entra-token",
+            num_retries=0,
+        )
+        result: Final = _anthropic(gateway).messages.count_tokens(model=model, messages=_MESSAGES)
+        assert result.input_tokens == 61
+        assert len(wire.drain()) == 1
+
+
+def test_anthropic_count_request_errors_use_the_anthropic_envelope(keyless_proxy: OwnedProxy) -> None:
+    gateway: Final = keyless_proxy.gateway
+    upstream_error: Final = {"type": "error", "error": {"type": "invalid_request_error", "message": "scripted reject"}}
+    with (
+        wire_server(lambda _request: Reply(status=400, body=json.dumps(upstream_error).encode())) as wire,
+        gateway.scenario() as scenario,
+    ):
+        model: Final = scenario.model(
+            model="anthropic/claude-sonnet-4-6", api_base=wire.url, api_key="synthetic-key", num_retries=0
+        )
+        missing_messages: Final = gateway.request("POST", "/v1/messages/count_tokens", {"model": model})
+        assert missing_messages.status_code == 400, missing_messages.text
+        assert _JSON_OBJECT.validate_json(missing_messages.content) == {
+            "type": "error",
+            "error": {"type": "invalid_request_error", "message": "messages parameter is required"},
+        }
+
+        rejected: Final = gateway.request(
+            "POST", "/v1/messages/count_tokens", {"model": model, "messages": _MESSAGES}
+        )
+        assert rejected.status_code == 400, rejected.text
+        body: Final = _JSON_OBJECT.validate_json(rejected.content)
+        assert set(body) <= {"type", "error", "request_id"}, body
+        assert body["error"] == upstream_error["error"], body
+        assert len(wire.drain()) == 1
+
+
+def test_openai_input_tokens_without_a_provider_key_is_an_openai_401(keyless_proxy: OwnedProxy) -> None:
+    gateway: Final = keyless_proxy.gateway
+    with wire_server(_never_called) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="openai/gpt-4o-mini", api_base=f"{wire.url}/v1", api_key=None, num_retries=0
+        )
+        client: Final = openai.OpenAI(
+            api_key=gateway.key, base_url=str(gateway.client.base_url), max_retries=0, http_client=gateway.client
+        )
+        with pytest.raises(openai.AuthenticationError) as raised:
+            client.responses.input_tokens.count(model=model, input="Count this synthetic request.")
+        assert f"No OpenAI API key is set for model {model}" in raised.value.message
+        assert wire.drain() == ()
+
+
+def test_google_count_tokens_errors_use_the_google_envelope(keyless_proxy: OwnedProxy) -> None:
+    gateway: Final = keyless_proxy.gateway
+    upstream_error: Final = {
+        "error": {"code": 400, "message": "API key not valid. Please pass a valid API key.", "status": "INVALID_ARGUMENT"}
+    }
+    with (
+        wire_server(lambda _request: Reply(status=400, body=json.dumps(upstream_error).encode())) as wire,
+        gateway.scenario() as scenario,
+    ):
+        model: Final = scenario.model(
+            model="gemini/gemini-2.5-flash", api_base=wire.url, api_key="synthetic-key", num_retries=0
+        )
+        rejected: Final = gateway.request(
+            "POST",
+            f"/v1beta/models/{model}:countTokens",
+            {"contents": [{"role": "user", "parts": [{"text": "Count this synthetic request."}]}]},
+        )
+        assert rejected.status_code == 400, rejected.text
+        error: Final = _JSON_OBJECT.validate_json(rejected.content)["error"]
+        assert isinstance(error, dict) and error["code"] == 400 and error["status"] == "INVALID_ARGUMENT", error
+        assert "API key not valid" in str(error["message"]), error
+        assert len(wire.drain()) == 1
+
+        malformed: Final = gateway.request("POST", f"/v1beta/models/{model}:countTokens", {"contents": "not-a-list"})
+        assert malformed.status_code == 400, malformed.text
+        assert _JSON_OBJECT.validate_json(malformed.content) == {
+            "error": {"code": 400, "message": "Invalid Google countTokens request body", "status": "INVALID_ARGUMENT"}
+        }

tests/integration/providers/test_custom_prompt_template_chaos.py

@@ -0,0 +1,542 @@
+import asyncio
+import json
+import re
+import threading
+import uuid
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass
+from pathlib import Path
+from queue import SimpleQueue
+from types import MappingProxyType
+from typing import Final, Literal
+from urllib.parse import urlsplit
+
+import httpx
+import psutil
+import pytest
+import yaml
+from integration._support.client import Gateway, eventually
+from integration._support.database import read_rows
+from integration._support.process import graceful_stop_seconds, owned_proxy_process
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+_BACKEND: Final = "llama3-custom-template-chaos"
+_API_KEY: Final = "synthetic-ollama-key"
+_CONFIG_MODEL: Final = "ollama-custom-template-chaos"
+_SYSTEM: Final = "Answer in one word."
+_ROLES: Final[dict[str, JsonValue]] = {
+    "system": {"pre_message": "<sys>", "post_message": "</sys>"},
+    "user": {"pre_message": "<user>", "post_message": "</user>"},
+}
+_REJECTION: Final = "initial_prompt_value: Input should be a valid string"
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_MARKER: Final = re.compile(r"marker-([0-9a-f]{32})")
+_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]")
+_OWNED_PROXY_CELL_SECONDS: Final = 2 * graceful_stop_seconds() + 120
+
+Endpoint = Literal["chat", "messages", "responses"]
+
+
+@dataclass(frozen=True, slots=True)
+class _Call:
+    endpoint: Endpoint
+    stream: bool
+    marker: str
+    rejected: bool = False
+
+
+@dataclass(frozen=True, slots=True)
+class _Served:
+    call: _Call
+    status: int
+    text: str
+
+
+def _question(marker: str) -> str:
+    return f"What colour is the sky? marker-{marker}"
+
+
+def _prompt(marker: str) -> str:
+    return f"<sys>{_SYSTEM}</sys><user>{_question(marker)}</user>"
+
+
+def _path(endpoint: Endpoint) -> str:
+    match endpoint:
+        case "chat":
+            return "/v1/chat/completions"
+        case "messages":
+            return "/v1/messages"
+        case "responses":
+            return "/v1/responses"
+
+
+def _body(model: str, call: _Call) -> dict[str, JsonValue]:
+    common: Final[dict[str, JsonValue]] = {
+        "model": model,
+        "stream": call.stream,
+        "num_retries": 0,
+        "cache": {"no-cache": True},
+        **({"initial_prompt_value": 5} if call.rejected else {}),
+    }
+    match call.endpoint:
+        case "chat":
+            return {
+                **common,
+                "messages": [
+                    {"role": "system", "content": _SYSTEM},
+                    {"role": "user", "content": _question(call.marker)},
+                ],
+            }
+        case "messages":
+            return {
+                **common,
+                "max_tokens": 64,
+                "system": _SYSTEM,
+                "messages": [{"role": "user", "content": _question(call.marker)}],
+            }
+        case "responses":
+            return {**common, "store": False, "instructions": _SYSTEM, "input": _question(call.marker)}
+
+
+def _generate_reply(marker: str, stream: bool, drop_connection: bool = False) -> Reply:
+    done: Final = {
+        "model": _BACKEND,
+        "created_at": "2026-10-15T00:00:00Z",
+        "response": "",
+        "done": True,
+        "done_reason": "stop",
+        "prompt_eval_count": 30,
+        "eval_count": 12,
+    }
+    if not stream:
+        return Reply(
+            body=json.dumps({**done, "response": f"blue marker-{marker}"}).encode(), drop_connection=drop_connection
+        )
+    frames: Final = (
+        *(
+            {"model": _BACKEND, "created_at": "2026-10-15T00:00:00Z", "response": piece, "done": False}
+            for piece in ("blue ", f"marker-{marker}")
+        ),
+        done,
+    )
+    return Reply(
+        content_type="application/x-ndjson",
+        chunks=tuple(json.dumps(frame).encode() + b"\n" for frame in frames),
+        drop_connection=drop_connection,
+    )
+
+
+def _is_generate(request: Request) -> bool:
+    return (request.method, request.target) == ("POST", "/api/generate")
+
+
+def _marker_of(request: Request) -> str:
+    found: Final = _MARKER.search(request.body.decode())
+    assert found is not None, request.body
+    return found.group(1)
+
+
+def _echo(request: Request) -> Reply:
+    if not _is_generate(request):
+        return Reply(body=b"{}")
+    stream: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True
+    return _generate_reply(_marker_of(request), stream)
+
+
+def _assert_each_prompt_rendered_once(received: tuple[Request, ...], markers: frozenset[str]) -> None:
+    generates: Final = tuple(request for request in received if _is_generate(request))
+    assert sorted(_marker_of(request) for request in generates) == sorted(markers)
+    for request in generates:
+        body: Final = _JSON_OBJECT.validate_json(request.body)
+        assert body["prompt"] == _prompt(_marker_of(request)), body["prompt"]
+        assert "roles" not in body and "initial_prompt_value" not in body, sorted(body)
+
+
+def _response_id(served: _Served) -> str | None:
+    if served.call.endpoint == "responses":
+        return None
+    if not served.call.stream:
+        identity: Final = _JSON_OBJECT.validate_json(served.text)["id"]
+        assert isinstance(identity, str)
+        return identity
+    for line in served.text.splitlines():
+        if not line.startswith("data: ") or line == "data: [DONE]":
+            continue
+        payload: Final = _JSON_OBJECT.validate_json(line.removeprefix("data: "))
+        if served.call.endpoint == "chat":
+            first: Final = payload["id"]
+            assert isinstance(first, str)
+            return first
+        if payload.get("type") == "message_start":
+            message: Final = payload["message"]
+            assert isinstance(message, dict) and isinstance(message["id"], str)
+            return message["id"]
+    raise AssertionError(served.text)
+
+
+def _spend_statuses(model: str, expected: int) -> MappingProxyType[str, str]:
+    rows: Final = eventually(
+        lambda: read_rows('SELECT request_id, status FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)),
+        lambda found: len(found) >= expected,
+        seconds=70,
+    )
+    statuses: Final = MappingProxyType({str(row["request_id"]): str(row["status"]) for row in rows})
+    assert len(statuses) == len(rows) == expected, rows
+    return statuses
+
+
+def _successes(statuses: Mapping[str, str]) -> frozenset[str]:
+    return frozenset(identity for identity, status in statuses.items() if status == "success")
+
+
+async def _send(client: httpx.AsyncClient, key: str, model: str, call: _Call) -> _Served:
+    async with client.stream(
+        "POST",
+        _path(call.endpoint),
+        json=_body(model, call),
+        headers={"Authorization": f"Bearer {key}", "anthropic-version": "2023-06-01"},
+    ) as response:
+        raw: Final = await response.aread()
+    return _Served(call=call, status=response.status_code, text=raw.decode())
+
+
+async def _burst(
+    base_url: str, key: str, model: str, calls: tuple[_Call, ...], *, tolerate_transport_errors: bool = False
+) -> tuple[_Served, ...]:
+    async with httpx.AsyncClient(base_url=base_url, timeout=60, trust_env=False) as client:
+        results: Final = await asyncio.gather(
+            *(_send(client, key, model, call) for call in calls), return_exceptions=tolerate_transport_errors
+        )
+    for result in results:
+        assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result)
+    return tuple(result for result in results if isinstance(result, _Served))
+
+
+def _calls(count: int, endpoints: tuple[Endpoint, ...], stream: Callable[[int], bool]) -> tuple[_Call, ...]:
+    return tuple(
+        _Call(endpoint=endpoints[index % len(endpoints)], stream=stream(index), marker=uuid.uuid4().hex)
+        for index in range(count)
+    )
+
+
+def _assert_answered_with_its_own_marker(served: _Served) -> None:
+    assert served.status == 200, served.text
+    assert set(_MARKER.findall(served.text)) == {served.call.marker}, served.text
+
+
+def _assert_rejected_by_name(served: _Served) -> None:
+    assert served.status == 500, served.text
+    assert _REJECTION in served.text and "marker-" not in served.text, served.text
+
+
+async def test_concurrent_burst_renders_each_template_and_rejects_only_the_bad_requests(gateway: Gateway) -> None:
+    calls: Final = tuple(
+        _Call(endpoint=call.endpoint, stream=call.stream, marker=call.marker, rejected=index % 5 == 4)
+        for index, call in enumerate(_calls(30, ("chat", "messages", "responses"), lambda index: index % 2 == 0))
+    )
+    accepted: Final = tuple(call for call in calls if not call.rejected)
+    with wire_server(_echo) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY, roles=_ROLES)
+        served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls)
+        assert len(served) == 30
+        for item in served:
+            if item.call.rejected:
+                _assert_rejected_by_name(item)
+            else:
+                _assert_answered_with_its_own_marker(item)
+        _assert_each_prompt_rendered_once(wire.drain(), frozenset(call.marker for call in accepted))
+        answered: Final = tuple(item for item in served if not item.call.rejected)
+        known: Final = frozenset(identity for identity in map(_response_id, answered) if identity is not None)
+        assert len(known) == 16, known
+        statuses: Final = _spend_statuses(model, 30)
+        assert len(_successes(statuses)) == len(accepted), statuses
+        assert known <= _successes(statuses)
+
+
+async def test_dropped_ollama_connections_fail_their_callers_and_the_rest_keep_their_template(
+    gateway: Gateway,
+) -> None:
+    calls: Final = _calls(12, ("chat",), lambda index: index % 2 == 1)
+    dropped: Final = frozenset(call.marker for index, call in enumerate(calls) if index % 3 == 0)
+
+    def respond(request: Request) -> Reply:
+        if not _is_generate(request):
+            return Reply(body=b"{}")
+        marker: Final = _marker_of(request)
+        stream: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True
+        return _generate_reply(marker, stream, drop_connection=marker in dropped)
+
+    with wire_server(respond) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY, roles=_ROLES)
+        served: Final = await _burst(str(gateway.client.base_url), gateway.key, model, calls)
+        assert len(served) == 12
+        for item in served:
+            if item.call.marker in dropped:
+                assert item.status == 500, item.text
+                assert "APIConnectionError" in item.text and "marker-" not in item.text, item.text
+            else:
+                _assert_answered_with_its_own_marker(item)
+        recovery: Final = _Call(endpoint="chat", stream=False, marker=uuid.uuid4().hex)
+        (recovered,) = await _burst(str(gateway.client.base_url), gateway.key, model, (recovery,))
+        _assert_answered_with_its_own_marker(recovered)
+        _assert_each_prompt_rendered_once(wire.drain(), frozenset(call.marker for call in (*calls, recovery)))
+        answered: Final = tuple(item for item in served if item.call.marker not in dropped)
+        survivors: Final = frozenset(
+            identity for identity in map(_response_id, (*answered, recovered)) if identity is not None
+        )
+        assert len(survivors) == 9, survivors
+        statuses: Final = _spend_statuses(model, 13)
+        assert _successes(statuses) == survivors, statuses
+        assert sum(status == "failure" for status in statuses.values()) == len(dropped), statuses
+
+
+def _owned_config(
+    wires: tuple[Wire, ...],
+    tmp_path: Path,
+    template: Mapping[str, JsonValue],
+    litellm_settings: Mapping[str, JsonValue],
+    router_settings: Mapping[str, JsonValue] = MappingProxyType({}),
+) -> Path:
+    base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
+    deployments: Final = [
+        {
+            "model_name": _CONFIG_MODEL,
+            "litellm_params": {
+                "model": f"ollama/{_BACKEND}",
+                "api_base": wire.url,
+                "api_key": _API_KEY,
+                "order": order,
+                **template,
+            },
+        }
+        for order, wire in enumerate(wires, start=1)
+    ]
+    config: Final = {
+        **base,
+        "model_list": deployments,
+        "litellm_settings": {**base.get("litellm_settings", {}), **litellm_settings},
+        "router_settings": {**base.get("router_settings", {}), **router_settings},
+    }
+    path: Final = tmp_path / "ollama-custom-template-chaos.yaml"
+    path.write_text(yaml.safe_dump(config))
+    return path
+
+
+def _open_upstream_connections(pid: int, upstream: str) -> int:
+    port: Final = urlsplit(upstream).port
+    return sum(
+        1
+        for connection in psutil.Process(pid).net_connections(kind="tcp")
+        if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port
+    )
+
+
+@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
+async def test_worker_sigkill_mid_burst_leaves_the_sibling_rendering_templates(
+    gateway: Gateway, tmp_path: Path
+) -> None:
+    calls: Final = _calls(20, ("chat",), lambda _: False)
+    release: Final = threading.Event()
+    held_markers: Final[SimpleQueue[str]] = SimpleQueue()
+
+    def held(request: Request) -> Reply:
+        if not _is_generate(request):
+            return Reply(body=b"{}")
+        held_markers.put(_marker_of(request))
+        assert release.wait(timeout=60), "The burst was never released"
+        return _echo(request)
+
+    with wire_server(held) as wire:
+        path: Final = _owned_config((wire,), tmp_path, {"roles": _ROLES}, {})
+        with owned_proxy_process(gateway, tmp_path, {}, config=path, workers=2) as owned:
+            candidate: Final = owned.gateway
+            workers: Final = eventually(
+                lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())),
+                lambda pids: len(pids) == 2,
+                seconds=30,
+            )
+            burst: Final = asyncio.create_task(
+                _burst(
+                    str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, calls, tolerate_transport_errors=True
+                )
+            )
+            await asyncio.to_thread(eventually, held_markers.qsize, lambda size: size == 20, 60)
+            held_by: Final = MappingProxyType({pid: _open_upstream_connections(pid, wire.url) for pid in workers})
+            assert sum(held_by.values()) == 20, held_by
+            victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__)
+            with owned.replacing_worker(victim_pid) as kill:
+                kill.freeze()
+                await asyncio.to_thread(kill)
+                release.set()
+                served: Final = await burst
+                assert held_by[survivor_pid] >= 10, held_by
+                assert len(served) == held_by[survivor_pid], (held_by, len(served))
+                for item in served:
+                    _assert_answered_with_its_own_marker(item)
+            follow_up: Final = _Call(endpoint="chat", stream=False, marker=uuid.uuid4().hex)
+            (answered,) = await _burst(str(candidate.client.base_url), candidate.key, _CONFIG_MODEL, (follow_up,))
+            _assert_answered_with_its_own_marker(answered)
+            _assert_each_prompt_rendered_once(wire.drain(), frozenset(call.marker for call in (*calls, follow_up)))
+
+
+@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
+async def test_rejected_templates_never_cool_down_the_preferred_deployment(gateway: Gateway, tmp_path: Path) -> None:
+    rejected: Final = tuple(
+        _Call(endpoint="chat", stream=False, marker=uuid.uuid4().hex, rejected=True) for _ in range(4)
+    )
+    valid: Final = _calls(8, ("chat",), lambda _: False)
+    with wire_server(_echo) as preferred, wire_server(_echo) as standby:
+        path: Final = _owned_config(
+            (preferred, standby), tmp_path, {"roles": _ROLES}, {}, {"disable_cooldowns": False, "allowed_fails": 0}
+        )
+        with owned_proxy_process(gateway, tmp_path, {}, config=path, workers=2) as owned:
+            base_url: Final = str(owned.gateway.client.base_url)
+            for served in await _burst(base_url, owned.gateway.key, _CONFIG_MODEL, rejected):
+                _assert_rejected_by_name(served)
+            for served in await _burst(base_url, owned.gateway.key, _CONFIG_MODEL, valid):
+                _assert_answered_with_its_own_marker(served)
+            _assert_each_prompt_rendered_once(preferred.drain(), frozenset(call.marker for call in valid))
+            assert not any(_is_generate(request) for request in standby.drain())
+
+
+def _cached_chat(candidate: Gateway, marker: str, field: str, value: str) -> httpx.Response:
+    return candidate.request(
+        "POST",
+        "/v1/chat/completions",
+        {
+            "model": _CONFIG_MODEL,
+            "messages": [{"role": "system", "content": _SYSTEM}, {"role": "user", "content": _question(marker)}],
+            field: value,
+        },
+    )
+
+
+def _served_from_cache(candidate: Gateway, field: str) -> bool:
+    marker: Final = uuid.uuid4().hex
+    first: Final = _cached_chat(candidate, marker, field, "<first>")
+    assert first.status_code == 200, first.text
+    assert "x-litellm-cache-key" not in first.headers, sorted(first.headers)
+    eventually(
+        lambda: _cached_chat(candidate, marker, field, "<first>"),
+        lambda found: "x-litellm-cache-key" in found.headers,
+        seconds=20,
+    )
+    changed: Final = _cached_chat(candidate, marker, field, "<second>")
+    assert changed.status_code == 200, changed.text
+    return "x-litellm-cache-key" in changed.headers
+
+
+@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
+def test_initial_and_final_prompt_values_key_the_provider_specific_cache_alike(
+    gateway: Gateway, tmp_path: Path
+) -> None:
+    template: Final[dict[str, JsonValue]] = {
+        "roles": _ROLES,
+        "initial_prompt_value": "<i>",
+        "final_prompt_value": "<f>",
+    }
+    with wire_server(_echo) as wire:
+        path: Final = _owned_config(
+            (wire,), tmp_path, template, {"enable_caching_on_provider_specific_optional_params": True}
+        )
+        with owned_proxy_process(gateway, tmp_path, {}, config=path, workers=2) as owned:
+            initial: Final = _served_from_cache(owned.gateway, "initial_prompt_value")
+            final: Final = _served_from_cache(owned.gateway, "final_prompt_value")
+            assert initial == final, {"initial_prompt_value": initial, "final_prompt_value": final}
+
+
+def _gemini_reply(_: Request) -> Reply:
+    candidate: Final[dict[str, JsonValue]] = {
+        "content": {"role": "model", "parts": [{"text": '{"colour": "blue"}'}]},
+        "finishReason": "STOP",
+    }
+    usage: Final[dict[str, JsonValue]] = {"promptTokenCount": 30, "candidatesTokenCount": 12, "totalTokenCount": 42}
+    return Reply(body=json.dumps({"candidates": [candidate], "usageMetadata": usage}).encode())
+
+
+def _texts(contents: JsonValue) -> tuple[str, ...]:
+    assert isinstance(contents, list), contents
+    parts: Final = tuple(part for content in contents if isinstance(content, dict) for part in _parts(content))
+    return tuple(text for part in parts if isinstance(text := part.get("text"), str))
+
+
+def _parts(content: dict[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]:
+    parts: Final = content.get("parts")
+    assert isinstance(parts, list), content
+    return tuple(part for part in parts if isinstance(part, dict))
+
+
+def _anthropic_text_reply(_: Request) -> Reply:
+    completion: Final[dict[str, JsonValue]] = {
+        "type": "completion",
+        "id": "compl_scripted",
+        "completion": "blue",
+        "stop_reason": "stop_sequence",
+    }
+    return Reply(body=json.dumps(completion).encode())
+
+
+def _boot_deployment(name: str, model: str, wire: Wire) -> dict[str, JsonValue]:
+    return {"model_name": name, "litellm_params": {"model": model, "api_base": wire.url, "api_key": _API_KEY}}
+
+
+def _boot_chat(candidate: Gateway, model: str, extra: Mapping[str, JsonValue]) -> None:
+    response: Final = candidate.request(
+        "POST",
+        "/v1/chat/completions",
+        {
+            "model": model,
+            "messages": [{"role": "system", "content": _SYSTEM}, {"role": "user", "content": _question(model)}],
+            "cache": {"no-cache": True},
+            **extra,
+        },
+    )
+    assert response.status_code == 200, response.text
+
+
+def _anthropic_text_prompts(wire: Wire) -> MappingProxyType[str, JsonValue]:
+    bodies: Final = tuple(_JSON_OBJECT.validate_json(request.body) for request in wire.drain())
+    return MappingProxyType({str(body["model"]): body["prompt"] for body in bodies})
+
+
+@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS)
+def test_boot_time_custom_prompt_dict_renders_partial_templates(gateway: Gateway, tmp_path: Path) -> None:
+    templates: Final[dict[str, JsonValue]] = {
+        "response_schema_prompt": {"roles": {"user": {"pre_message": "<schema>", "post_message": "</schema>"}}},
+        "claude-2.1": {"roles": _ROLES},
+        "claude-instant-1.2": {"initial_prompt_value": "<i>", "final_prompt_value": "<f>"},
+    }
+    schema: Final[dict[str, JsonValue]] = {
+        "type": "object",
+        "properties": {"colour": {"type": "string"}},
+        "required": ["colour"],
+    }
+    with wire_server(_gemini_reply) as gemini, wire_server(_anthropic_text_reply) as anthropic_text:
+        base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
+        deployments: Final = [
+            _boot_deployment("gemini-schema", "gemini/scripted-template-model", gemini),
+            _boot_deployment("anthropic-text-roles", "anthropic_text/claude-2.1", anthropic_text),
+            _boot_deployment("anthropic-text-values", "anthropic_text/claude-instant-1.2", anthropic_text),
+        ]
+        settings: Final = {**base.get("litellm_settings", {}), "custom_prompt_dict": templates}
+        path: Final = tmp_path / "boot-time-custom-prompt-dict.yaml"
+        path.write_text(yaml.safe_dump({**base, "model_list": deployments, "litellm_settings": settings}))
+        with owned_proxy_process(gateway, tmp_path, {}, config=path, workers=2) as owned:
+            _boot_chat(
+                owned.gateway,
+                "gemini-schema",
+                {"response_format": {"type": "json_schema", "json_schema": {"name": "sky", "schema": schema}}},
+            )
+            posts: Final = tuple(request for request in gemini.drain() if request.method == "POST")
+            assert len(posts) == 1, [request.target for request in posts]
+            texts: Final = _texts(_JSON_OBJECT.validate_json(posts[0].body)["contents"])
+            wrapped: Final = tuple(text for text in texts if text.startswith("<schema>"))
+            assert len(wrapped) == 1, texts
+            assert wrapped[0].endswith("</schema>") and "colour" in wrapped[0], wrapped[0]
+            _boot_chat(owned.gateway, "anthropic-text-roles", {})
+            _boot_chat(owned.gateway, "anthropic-text-values", {})
+            assert _anthropic_text_prompts(anthropic_text) == {
+                "claude-2.1": f"<sys>{_SYSTEM}</sys><user>{_question('anthropic-text-roles')}</user>",
+                "claude-instant-1.2": f"<i>{_SYSTEM}{_question('anthropic-text-values')}<f>",
+            }

tests/integration/providers/test_custom_prompt_template_wire.py

@@ -0,0 +1,701 @@
+import json
+import uuid
+from collections.abc import Callable, Iterator, Mapping
+from contextlib import contextmanager
+from dataclasses import dataclass
+from typing import Final
+
+import anthropic
+import openai
+import pytest
+from integration._support.client import Gateway, Scenario, eventually, object_value, string_value
+from integration._support.database import read_rows
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+_BACKEND: Final = "llama3-custom-template"
+_API_KEY: Final = "synthetic-ollama-key"
+_SYSTEM: Final = "Answer in one word."
+_QUESTION: Final = "What colour is the sky?"
+_ANSWER: Final = "blue"
+_INITIAL: Final = "<bos>"
+_FINAL: Final = "<assistant>"
+_ROLES: Final[dict[str, JsonValue]] = {
+    "system": {"pre_message": "<sys>", "post_message": "</sys>"},
+    "user": {"pre_message": "<user>", "post_message": "</user>"},
+    "assistant": {"pre_message": "<asst>", "post_message": "</asst>"},
+}
+_ROLES_ONLY: Final[dict[str, JsonValue]] = {"roles": _ROLES}
+_VALUES_ONLY: Final[dict[str, JsonValue]] = {"initial_prompt_value": _INITIAL, "final_prompt_value": _FINAL}
+_FULL: Final[dict[str, JsonValue]] = {**_ROLES_ONLY, **_VALUES_ONLY}
+_ROLES_PROMPT: Final = f"<sys>{_SYSTEM}</sys><user>{_QUESTION}</user>"
+_VALUES_PROMPT: Final = f"{_INITIAL}{_SYSTEM}{_QUESTION}{_FINAL}"
+_FULL_PROMPT: Final = f"{_INITIAL}{_ROLES_PROMPT}{_FINAL}"
+_TEMPLATES: Final[Mapping[str, tuple[dict[str, JsonValue], str]]] = {
+    "roles_only": (_ROLES_ONLY, _ROLES_PROMPT),
+    "values_only": (_VALUES_ONLY, _VALUES_PROMPT),
+}
+_TEMPLATE_FIELDS: Final = frozenset({"roles", "initial_prompt_value", "final_prompt_value", "bos_token", "eos_token"})
+_AWS: Final[dict[str, JsonValue]] = {
+    "aws_access_key_id": "AKIASYNTHETICKEY0001",
+    "aws_secret_access_key": "synthetic-secret-key-for-testing",
+    "aws_region_name": "us-east-1",
+}
+_NO_CACHE: Final[dict[str, JsonValue]] = {"no-cache": True}
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_JSON_LIST: Final = TypeAdapter(list[JsonValue])
+
+
+def _generate_reply(text: str) -> Reply:
+    return Reply(
+        body=json.dumps(
+            {
+                "model": _BACKEND,
+                "created_at": "2026-10-15T00:00:00Z",
+                "response": text,
+                "done": True,
+                "done_reason": "stop",
+                "prompt_eval_count": 30,
+                "eval_count": 12,
+            }
+        ).encode()
+    )
+
+
+def _streamed_reply(text: str) -> Reply:
+    frames: Final = (
+        {"model": _BACKEND, "created_at": "2026-10-15T00:00:00Z", "response": text, "done": False},
+        {
+            "model": _BACKEND,
+            "created_at": "2026-10-15T00:00:00Z",
+            "response": "",
+            "done": True,
+            "done_reason": "stop",
+            "prompt_eval_count": 30,
+            "eval_count": 12,
+        },
+    )
+    return Reply(
+        content_type="application/x-ndjson", chunks=tuple(json.dumps(frame).encode() + b"\n" for frame in frames)
+    )
+
+
+def _is_generate(request: Request) -> bool:
+    return (request.method, request.target) == ("POST", "/api/generate")
+
+
+def _echo_answer(request: Request) -> Reply:
+    stream: Final = _JSON_OBJECT.validate_json(request.body).get("stream") is True
+    return _streamed_reply(_ANSWER) if stream else _generate_reply(_ANSWER)
+
+
+@contextmanager
+def _ollama_server(respond: Callable[[Request], Reply] = _echo_answer) -> Iterator[Wire]:
+    with wire_server(lambda request: respond(request) if _is_generate(request) else Reply(body=b"{}")) as wire:
+        yield wire
+
+
+def _generate_bodies(wire: Wire) -> tuple[dict[str, JsonValue], ...]:
+    return tuple(_JSON_OBJECT.validate_json(request.body) for request in wire.drain() if _is_generate(request))
+
+
+def _assert_rendered(body: dict[str, JsonValue], prompt: str, *, stream: bool) -> None:
+    assert body["model"] == _BACKEND
+    assert body["prompt"] == prompt, body["prompt"]
+    assert body["stream"] is stream
+    options: Final = body.get("options", {})
+    assert isinstance(options, dict)
+    assert not _TEMPLATE_FIELDS & (set(body) | set(options)), sorted(body)
+
+
+def _assert_only_rendered(wire: Wire, prompt: str, *, stream: bool = False) -> None:
+    bodies: Final = _generate_bodies(wire)
+    assert len(bodies) == 1, bodies
+    _assert_rendered(bodies[0], prompt, stream=stream)
+
+
+def _ollama_model(scenario: Scenario, wire: Wire, template: Mapping[str, JsonValue]) -> str:
+    return scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY, **template)
+
+
+def _chat_messages() -> list[dict[str, JsonValue]]:
+    return [{"role": "system", "content": _SYSTEM}, {"role": "user", "content": _QUESTION}]
+
+
+def _spend_row(identity: str) -> dict[str, JsonValue]:
+    rows: Final = eventually(
+        lambda: read_rows(
+            'SELECT model_group, status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
+            (identity,),
+        ),
+        lambda found: len(found) == 1,
+        seconds=70,
+    )
+    return rows[0]
+
+
+def _model_spend_rows(model: str, expected: int) -> list[dict[str, JsonValue]]:
+    rows: Final = eventually(
+        lambda: read_rows(
+            'SELECT request_id, status, prompt_tokens, completion_tokens FROM "LiteLLM_SpendLogs" WHERE model_group=%s',
+            (model,),
+        ),
+        lambda found: len(found) >= expected,
+        seconds=70,
+    )
+    assert len({row["request_id"] for row in rows}) == len(rows) == expected, rows
+    return rows
+
+
+def _billed(model: str) -> dict[str, JsonValue]:
+    return {"model_group": model, "status": "success", "prompt_tokens": 30, "completion_tokens": 12}
+
+
+def _openai_client(gateway: Gateway) -> openai.OpenAI:
+    return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0)
+
+
+def _async_openai_client(gateway: Gateway) -> openai.AsyncOpenAI:
+    return openai.AsyncOpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0)
+
+
+def _anthropic_client(gateway: Gateway) -> anthropic.Anthropic:
+    return anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
+
+
+def _async_anthropic_client(gateway: Gateway) -> anthropic.AsyncAnthropic:
+    return anthropic.AsyncAnthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
+
+
+def _post(gateway: Gateway, path: str, body: Mapping[str, JsonValue]) -> tuple[int, str]:
+    response: Final = gateway.request("POST", path, {**body, "cache": _NO_CACHE})
+    return response.status_code, response.text
+
+
+def _chat(gateway: Gateway, model: str, **extra: JsonValue) -> tuple[int, str]:
+    return _post(gateway, "/v1/chat/completions", {"model": model, "messages": _chat_messages(), **extra})
+
+
+def _chat_content(text: str) -> str:
+    choices: Final = _JSON_LIST.validate_python(_JSON_OBJECT.validate_json(text)["choices"])
+    message: Final = object_value(object_value(choices[0])["message"])
+    return string_value(message["content"])
+
+
+def test_openai_sdk_chat_renders_a_roles_only_template(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _ROLES_ONLY)
+        completion: Final = _openai_client(gateway).chat.completions.create(
+            model=model,
+            messages=_chat_messages(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            extra_body={"cache": _NO_CACHE},
+        )
+        assert completion.choices[0].message.content == _ANSWER
+        assert completion.choices[0].finish_reason == "stop"
+        assert completion.usage is not None
+        assert (completion.usage.prompt_tokens, completion.usage.completion_tokens) == (30, 12)
+        _assert_only_rendered(wire, _ROLES_PROMPT)
+        assert _spend_row(completion.id) == _billed(model)
+
+
+async def test_async_openai_sdk_chat_stream_renders_a_roles_only_template(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _ROLES_ONLY)
+        stream: Final = await _async_openai_client(gateway).chat.completions.create(
+            model=model,
+            messages=_chat_messages(),  # pyright: ignore[reportArgumentType]  # plain JSON messages
+            stream=True,
+            stream_options={"include_usage": True},
+            extra_body={"cache": _NO_CACHE},
+        )
+        chunks: Final = [chunk async for chunk in stream]
+        assert {chunk.id for chunk in chunks} == {chunks[0].id}
+        assert "".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices) == _ANSWER
+        usages: Final = [chunk.usage for chunk in chunks if chunk.usage is not None]
+        assert [(usage.prompt_tokens, usage.completion_tokens) for usage in usages] == [(30, 12)]
+        _assert_only_rendered(wire, _ROLES_PROMPT, stream=True)
+        assert _spend_row(chunks[0].id) == _billed(model)
+
+
+def test_openai_sdk_completions_render_a_roles_only_template(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _ROLES_ONLY)
+        completion: Final = _openai_client(gateway).completions.create(
+            model=model, prompt=_QUESTION, extra_body={"cache": _NO_CACHE}
+        )
+        assert completion.choices[0].text == _ANSWER
+        _assert_only_rendered(wire, f"<user>{_QUESTION}</user>")
+        assert _spend_row(completion.id) == _billed(model)
+
+
+def test_anthropic_sdk_messages_render_a_roles_only_template(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _ROLES_ONLY)
+        message: Final = _anthropic_client(gateway).messages.create(
+            model=model,
+            max_tokens=64,
+            system=_SYSTEM,
+            messages=[{"role": "user", "content": _QUESTION}],
+            extra_body={"cache": _NO_CACHE},
+        )
+        assert message.stop_reason == "end_turn"
+        assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", _ANSWER)]
+        assert (message.usage.input_tokens, message.usage.output_tokens) == (30, 12)
+        _assert_only_rendered(wire, _ROLES_PROMPT)
+        assert _spend_row(message.id) == _billed(model)
+
+
+async def test_async_anthropic_sdk_messages_stream_render_a_roles_only_template(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _ROLES_ONLY)
+        async with _async_anthropic_client(gateway).messages.stream(
+            model=model,
+            max_tokens=64,
+            system=_SYSTEM,
+            messages=[{"role": "user", "content": _QUESTION}],
+            extra_body={"cache": _NO_CACHE},
+        ) as stream:
+            text: Final = "".join([piece async for piece in stream.text_stream])
+            final: Final = await stream.get_final_message()
+        assert text == _ANSWER
+        assert final.stop_reason == "end_turn"
+        _assert_only_rendered(wire, _ROLES_PROMPT, stream=True)
+        assert _spend_row(final.id) == _billed(model)
+
+
+def test_openai_sdk_responses_render_a_roles_only_template(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _ROLES_ONLY)
+        response: Final = _openai_client(gateway).responses.create(
+            model=model, instructions=_SYSTEM, input=_QUESTION, store=False, extra_body={"cache": _NO_CACHE}
+        )
+        assert response.output_text == _ANSWER
+        assert response.usage is not None and (response.usage.input_tokens, response.usage.output_tokens) == (30, 12)
+        _assert_only_rendered(wire, _ROLES_PROMPT)
+        rows: Final = _model_spend_rows(model, 1)
+        assert (rows[0]["status"], rows[0]["prompt_tokens"], rows[0]["completion_tokens"]) == ("success", 30, 12)
+
+
+async def test_async_openai_sdk_responses_stream_render_a_roles_only_template(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _ROLES_ONLY)
+        stream: Final = await _async_openai_client(gateway).responses.create(
+            model=model,
+            instructions=_SYSTEM,
+            input=_QUESTION,
+            store=False,
+            stream=True,
+            extra_body={"cache": _NO_CACHE},
+        )
+        events: Final = [event async for event in stream]
+        assert "".join(event.delta for event in events if event.type == "response.output_text.delta") == _ANSWER
+        completed: Final = [event for event in events if event.type == "response.completed"]
+        assert len(completed) == 1 and completed[0].response.output_text == _ANSWER
+        _assert_only_rendered(wire, _ROLES_PROMPT, stream=True)
+        assert _model_spend_rows(model, 1)[0]["status"] == "success"
+
+
+def test_prompt_values_only_template_wraps_the_bare_messages(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _VALUES_ONLY)
+        code, text = _chat(gateway, model)
+        assert code == 200, text
+        assert _chat_content(text) == _ANSWER
+        _assert_only_rendered(wire, _VALUES_PROMPT)
+
+
+def test_full_template_renders_every_part(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _FULL)
+        code, text = _chat(gateway, model)
+        assert code == 200, text
+        assert _chat_content(text) == _ANSWER
+        _assert_only_rendered(wire, _FULL_PROMPT)
+
+
+@dataclass(frozen=True, slots=True)
+class _Member:
+    deployment: Callable[[str], dict[str, JsonValue]]
+    reply: Callable[[str], Reply]
+    prompt_field: str
+    target: Callable[[str], bool]
+
+
+def _codestral_reply(text: str) -> Reply:
+    return Reply(
+        body=json.dumps(
+            {
+                "id": "cmpl-scripted",
+                "object": "chat.completion",
+                "model": "codestral-latest",
+                "created": 1,
+                "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}],
+                "usage": {"prompt_tokens": 30, "completion_tokens": 12, "total_tokens": 42},
+            }
+        ).encode()
+    )
+
+
+def _predibase_reply(text: str) -> Reply:
+    return Reply(body=json.dumps({"generated_text": text, "details": {"finish_reason": "eos_token"}}).encode())
+
+
+def _sagemaker_reply(text: str) -> Reply:
+    return Reply(body=json.dumps([{"generated_text": text}]).encode())
+
+
+def _bedrock_meta_reply(text: str) -> Reply:
+    return Reply(
+        body=json.dumps(
+            {"generation": text, "prompt_token_count": 30, "generation_token_count": 12, "stop_reason": "stop"}
+        ).encode()
+    )
+
+
+_FAMILY: Final[Mapping[str, _Member]] = {
+    "codestral": _Member(
+        deployment=lambda url: {
+            "model": "text-completion-codestral/codestral-latest",
+            "api_base": f"{url}/v1/fim/completions",
+            "api_key": "scripted-key",
+        },
+        reply=_codestral_reply,
+        prompt_field="prompt",
+        target=lambda target: target == "/v1/fim/completions",
+    ),
+    "predibase": _Member(
+        deployment=lambda url: {
+            "model": "predibase/llama-3-8b-instruct",
+            "api_base": url,
+            "api_key": "scripted-key",
+            "predibase_tenant_id": "scripted-tenant",
+        },
+        reply=_predibase_reply,
+        prompt_field="inputs",
+        target=lambda target: target.startswith("/scripted-tenant/deployments/v2/llms/llama-3-8b-instruct/generate"),
+    ),
+    "sagemaker": _Member(
+        deployment=lambda url: {
+            "model": "sagemaker/scripted-endpoint",
+            "sagemaker_base_url": f"{url}/endpoints/scripted-endpoint/invocations",
+            **_AWS,
+        },
+        reply=_sagemaker_reply,
+        prompt_field="inputs",
+        target=lambda target: target == "/endpoints/scripted-endpoint/invocations",
+    ),
+    "bedrock_invoke": _Member(
+        deployment=lambda url: {"model": "bedrock/invoke/meta.llama3-8b-instruct-v1:0", "api_base": url, **_AWS},
+        reply=_bedrock_meta_reply,
+        prompt_field="prompt",
+        target=lambda target: target.endswith("/invoke"),
+    ),
+}
+
+
+@pytest.mark.parametrize("template", sorted(_TEMPLATES))
+@pytest.mark.parametrize("member", sorted(_FAMILY))
+def test_family_member_renders_a_partial_template(gateway: Gateway, member: str, template: str) -> None:
+    spec: Final = _FAMILY[member]
+    fields, prompt = _TEMPLATES[template]
+    with wire_server(lambda _: spec.reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(**spec.deployment(wire.url), **fields)
+        code, text = _chat(gateway, model)
+        received: Final = tuple(request for request in wire.drain() if request.method == "POST")
+        assert code == 200, text
+        assert _chat_content(text) == _ANSWER
+        assert len(received) == 1, [request.target for request in received]
+        assert spec.target(received[0].target), received[0].target
+        body: Final = _JSON_OBJECT.validate_json(received[0].body)
+        assert body[spec.prompt_field] == prompt, body
+        assert not _TEMPLATE_FIELDS & set(body), sorted(body)
+
+
+@pytest.mark.parametrize("provider", ("vllm", "petals"))
+def test_in_process_family_member_fails_alone_and_the_rig_keeps_serving(gateway: Gateway, provider: str) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        broken: Final = scenario.model(model=f"{provider}/scripted-model", **_ROLES_ONLY)
+        healthy: Final = _ollama_model(scenario, wire, _ROLES_ONLY)
+        failed, failure = _chat(gateway, broken)
+        assert failed >= 400, failure
+        code, text = _chat(gateway, healthy)
+        assert code == 200, text
+        _assert_only_rendered(wire, _ROLES_PROMPT)
+
+
+def _watsonx_reply(text: str) -> Reply:
+    result: Final[dict[str, JsonValue]] = {
+        "generated_text": text,
+        "generated_token_count": 12,
+        "input_token_count": 30,
+        "stop_reason": "eos_token",
+    }
+    return Reply(body=json.dumps({"model_id": "ibm/granite", "results": [result]}).encode())
+
+
+def test_watsonx_text_deployment_with_a_template_still_answers(gateway: Gateway) -> None:
+    with wire_server(lambda _: _watsonx_reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(
+            model="watsonx_text/ibm/granite-chat-scripted",
+            api_base=wire.url,
+            token="scripted-token",
+            project_id="p",
+            **_ROLES_ONLY,
+        )
+        code, text = _chat(gateway, model)
+        assert code == 200, text
+        assert _chat_content(text) == _ANSWER
+        posts: Final = tuple(request for request in wire.drain() if request.method == "POST")
+        assert len(posts) == 1, [request.target for request in posts]
+        body: Final = _JSON_OBJECT.validate_json(posts[0].body)
+        assert _QUESTION in string_value(body["input"]), body
+        assert not _TEMPLATE_FIELDS & set(body), sorted(body)
+
+
+def _openai_chat_reply(text: str) -> Reply:
+    return Reply(
+        body=json.dumps(
+            {
+                "id": "chatcmpl-scripted",
+                "object": "chat.completion",
+                "created": 1,
+                "model": "scripted-compat",
+                "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}],
+                "usage": {"prompt_tokens": 30, "completion_tokens": 12, "total_tokens": 42},
+            }
+        ).encode()
+    )
+
+
+def _converse_reply(text: str) -> Reply:
+    return Reply(
+        body=json.dumps(
+            {
+                "output": {"message": {"role": "assistant", "content": [{"text": text}]}},
+                "stopReason": "end_turn",
+                "usage": {"inputTokens": 30, "outputTokens": 12, "totalTokens": 42},
+                "metrics": {"latencyMs": 1},
+            }
+        ).encode()
+    )
+
+
+_FORWARDING: Final[Mapping[str, tuple[Callable[[str], dict[str, JsonValue]], Callable[[str], Reply]]]] = {
+    "openai_compatible": (
+        lambda url: {"model": "openai/scripted-compat", "api_base": f"{url}/v1", "api_key": "scripted-key"},
+        _openai_chat_reply,
+    ),
+    "bedrock_converse": (
+        lambda url: {"model": "bedrock/converse/meta.llama3-8b-instruct-v1:0", "api_base": url, **_AWS},
+        _converse_reply,
+    ),
+}
+
+
+def _keys_anywhere(value: JsonValue) -> frozenset[str]:
+    if isinstance(value, dict):
+        nested: Final = frozenset(key for child in value.values() for key in _keys_anywhere(child))
+        return frozenset(value) | nested
+    if isinstance(value, list):
+        return frozenset(key for child in value for key in _keys_anywhere(child))
+    return frozenset()
+
+
+@pytest.mark.parametrize("source", ("deployment", "request"))
+@pytest.mark.parametrize("member", sorted(_FORWARDING))
+def test_template_fields_never_reach_a_provider_that_does_not_render_them(
+    gateway: Gateway, member: str, source: str
+) -> None:
+    deployment, reply = _FORWARDING[member]
+    with wire_server(lambda _: reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        model: Final = scenario.model(**deployment(wire.url), **(_FULL if source == "deployment" else {}))
+        code, text = _chat(gateway, model, **(_FULL if source == "request" else {}))
+        assert code == 200, text
+        assert _chat_content(text) == _ANSWER
+        posts: Final = tuple(request for request in wire.drain() if request.method == "POST")
+        assert len(posts) == 1, [request.target for request in posts]
+        body: Final = _JSON_OBJECT.validate_json(posts[0].body)
+        assert not _TEMPLATE_FIELDS & _keys_anywhere(body), body
+        assert _INITIAL not in posts[0].body.decode() and _FINAL not in posts[0].body.decode(), body
+
+
+@pytest.mark.parametrize("value", (5, [_INITIAL]), ids=("int", "list"))
+@pytest.mark.parametrize("source", ("deployment", "request"))
+def test_non_string_initial_prompt_value_is_rejected_by_name(gateway: Gateway, source: str, value: JsonValue) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        template: Final[dict[str, JsonValue]] = {**_FULL, "initial_prompt_value": value}
+        broken: Final = _ollama_model(scenario, wire, template if source == "deployment" else _FULL)
+        healthy: Final = _ollama_model(scenario, wire, _FULL)
+        failed, failure = _chat(gateway, broken, **({"initial_prompt_value": value} if source == "request" else {}))
+        assert failed == 500, failure
+        assert "initial_prompt_value: Input should be a valid string" in failure, failure
+        assert _generate_bodies(wire) == ()
+        code, text = _chat(gateway, healthy)
+        assert code == 200, text
+        _assert_only_rendered(wire, _FULL_PROMPT)
+
+
+@pytest.mark.parametrize("value", ("", None), ids=("empty", "null"))
+def test_blank_initial_prompt_value_in_the_request_drops_the_deployment_value(
+    gateway: Gateway, value: JsonValue
+) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _FULL)
+        code, text = _chat(gateway, model, initial_prompt_value=value)
+        assert code == 200, text
+        _assert_only_rendered(wire, f"{_ROLES_PROMPT}{_FINAL}")
+
+
+def test_request_initial_prompt_value_overrides_the_deployment_value(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _FULL)
+        code, text = _chat(gateway, model, initial_prompt_value="<override>")
+        assert code == 200, text
+        _assert_only_rendered(wire, f"<override>{_ROLES_PROMPT}{_FINAL}")
+
+
+def test_five_kilobyte_initial_prompt_value_leads_the_prompt(gateway: Gateway) -> None:
+    large: Final = "x" * 5120
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _ROLES_ONLY)
+        code, text = _chat(gateway, model, initial_prompt_value=large)
+        assert code == 200, text
+        _assert_only_rendered(wire, f"{large}{_ROLES_PROMPT}")
+
+
+def test_identical_requests_each_render_and_bill_once(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _ROLES_ONLY)
+        first, first_text = _chat(gateway, model)
+        second, second_text = _chat(gateway, model)
+        assert (first, second) == (200, 200), (first_text, second_text)
+        bodies: Final = _generate_bodies(wire)
+        assert len(bodies) == 2, bodies
+        for body in bodies:
+            _assert_rendered(body, _ROLES_PROMPT, stream=False)
+        identities: Final = {string_value(_JSON_OBJECT.validate_json(text)["id"]) for text in (first_text, second_text)}
+        assert len(identities) == 2
+        for identity in identities:
+            assert _spend_row(identity) == _billed(model)
+
+
+def test_template_with_only_a_bos_token_names_the_fields_it_lacks(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, {"bos_token": "<s>"})
+        code, text = _chat(gateway, model)
+        assert code == 500, text
+        assert "it sets none of roles, initial_prompt_value, or final_prompt_value" in text, text
+        assert _generate_bodies(wire) == ()
+
+
+def test_roles_that_are_not_a_mapping_render_without_role_markers(gateway: Gateway) -> None:
+    with _ollama_server() as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, {**_VALUES_ONLY, "roles": json.dumps(_ROLES)})
+        code, text = _chat(gateway, model)
+        assert code == 200, text
+        _assert_only_rendered(wire, _VALUES_PROMPT)
+
+
+def test_provider_error_reaches_the_caller_after_one_attempt(gateway: Gateway) -> None:
+    missing: Final = Reply(status=404, body=json.dumps({"error": f"model '{_BACKEND}' not found"}).encode())
+    with _ollama_server(lambda _: missing) as wire, gateway.scenario() as scenario:
+        model: Final = _ollama_model(scenario, wire, _ROLES_ONLY)
+        code, text = _chat(gateway, model)
+        assert code == 404, text
+        assert f"model '{_BACKEND}' not found" in text, text
+        bodies: Final = _generate_bodies(wire)
+        assert len(bodies) == 1, bodies
+        _assert_rendered(bodies[0], _ROLES_PROMPT, stream=False)
+
+
+def _template_keys(fields: Mapping[str, JsonValue]) -> tuple[bool, bool]:
+    return "initial_prompt_value" in fields, "final_prompt_value" in fields
+
+
+def test_pass_through_body_treats_initial_and_final_prompt_values_alike(gateway: Gateway) -> None:
+    with wire_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario:
+        path: Final = f"/integration-template-passthrough-{uuid.uuid4().hex}"
+        created: Final = gateway.post(
+            "/config/pass_through_endpoint", {"path": path, "target": f"{wire.url}/api/generate"}
+        )
+        endpoint: Final = object_value(_JSON_LIST.validate_python(created["endpoints"])[0])
+        scenario.cleanups.callback(
+            lambda: gateway.request(
+                "DELETE", "/config/pass_through_endpoint", params={"endpoint_id": str(endpoint["id"])}
+            )
+        )
+        response: Final = eventually(
+            lambda: gateway.request("POST", path, {"model": _BACKEND, "prompt": _QUESTION, "stream": False, **_FULL}),
+            lambda answered: answered.status_code != 404,
+            seconds=30,
+        )
+        assert response.status_code == 200, response.text
+        posts: Final = tuple(request for request in wire.drain() if request.method == "POST")
+        forwarded: Final = _JSON_OBJECT.validate_json(posts[-1].body)
+        assert (forwarded["model"], forwarded["prompt"]) == (_BACKEND, _QUESTION), forwarded
+        initial, final = _template_keys(forwarded)
+        assert initial == final, sorted(forwarded)
+
+
+def _a2a_reply(request: Request) -> Reply:
+    result: Final[dict[str, JsonValue]] = {
+        "kind": "message",
+        "role": "agent",
+        "messageId": f"{uuid.uuid4().hex}-out",
+        "parts": [{"kind": "text", "text": _ANSWER}],
+    }
+    outbound: Final = _JSON_OBJECT.validate_json(request.body)
+    return Reply(body=json.dumps({"jsonrpc": "2.0", "id": outbound["id"], "result": result}).encode())
+
+
+def test_a2a_params_treat_initial_and_final_prompt_values_alike(gateway: Gateway) -> None:
+    marker: Final = uuid.uuid4().hex
+    agent_name: Final = f"template-agent-{marker}"
+
+    def peer(request: Request) -> Reply:
+        if request.method == "GET":
+            return Reply(body=json.dumps(_agent_card(wire.url, agent_name)).encode())
+        return _a2a_reply(request)
+
+    with wire_server(peer) as wire, gateway.scenario() as scenario:
+        created: Final = gateway.post(
+            "/v1/agents", {"agent_name": agent_name, "agent_card_params": _agent_card(wire.url, agent_name)}
+        )
+        agent_id: Final = string_value(created["agent_id"])
+        scenario.cleanups.callback(lambda: gateway.request("DELETE", f"/v1/agents/{agent_id}"))
+        message: Final[dict[str, JsonValue]] = {
+            "kind": "message",
+            "role": "user",
+            "messageId": f"{marker}-in",
+            "parts": [{"kind": "text", "text": _QUESTION}],
+        }
+        invoked: Final = gateway.request(
+            "POST",
+            f"/a2a/{agent_id}",
+            {
+                "jsonrpc": "2.0",
+                "id": marker,
+                "method": "message/send",
+                "params": {"message": message, "initial_prompt_value": _INITIAL, "final_prompt_value": _FINAL},
+            },
+        )
+        assert invoked.status_code == 200, invoked.text
+        assert object_value(invoked.json()["result"])["parts"] == [{"kind": "text", "text": _ANSWER}], invoked.text
+        posts: Final = tuple(request for request in wire.drain() if request.method == "POST")
+        assert len(posts) == 1, [request.target for request in posts]
+        params: Final = object_value(_JSON_OBJECT.validate_json(posts[0].body)["params"])
+        assert _template_keys(params) == (False, False), sorted(params)
+
+
+def _agent_card(url: str, name: str) -> dict[str, JsonValue]:
+    return {
+        "protocolVersion": "0.3",
+        "name": name,
+        "description": "Scripted A2A peer",
+        "version": "1.0.0",
+        "url": url + "/",
+        "capabilities": {"streaming": True},
+        "defaultInputModes": ["text"],
+        "defaultOutputModes": ["text"],
+        "skills": [],
+    }

tests/integration/providers/test_gemini_anyof_null_branches_wire.py

@@ -0,0 +1,681 @@
+import asyncio
+import base64
+import json
+import re
+import threading
+import uuid
+from collections.abc import AsyncIterator, Iterator, Mapping
+from concurrent.futures import ThreadPoolExecutor
+from itertools import product
+from types import MappingProxyType
+from typing import Final, Literal
+
+import anthropic
+import httpx
+import openai
+import pytest
+import websockets
+from integration._support.client import (
+    JSON_OBJECT,
+    Gateway,
+    Scenario,
+    gateway_from_environment,
+    object_value,
+    string_value,
+)
+from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process
+from integration._support.vertex import service_account_json
+from integration._support.wire import Reply, Request, Wire, wire_server
+from integration._support.upstream import GEMINI_LIVE_PATH
+from integration.providers._gemini_tts_voice_support import live_deployment, live_scenario
+from integration.providers._vertex_batch_rows import encoded, gcs_peer, is_gcs_upload, jsonl, upload, vertex_model
+from pydantic import JsonValue, TypeAdapter
+
+Provider = Literal["vertex_ai", "gemini"]
+Endpoint = Literal["chat", "messages", "responses"]
+
+_BACKEND: Final = "gemini-3.8-flash"
+_NO_JSON_SCHEMA_BACKEND: Final = "gemini-flash-latest"
+_IMAGE_BACKEND: Final = "gemini-3.1-flash-image"
+_PROJECT: Final = "scripted-project"
+_LOCATION: Final = "us-central1"
+_GEMINI_KEY: Final = "sk-scripted-gemini"
+_COUNT: Final = 777
+_FAIL_MARKER: Final = "scripted-invalid-argument"
+_STRING_ANYOF: Final = re.compile(rb'"anyOf":\s*"')
+_SCALAR_BRANCH: Final = re.compile(rb'"anyOf":\s*\[\s*(?:true|false|-?\d)')
+_PNG: Final = base64.b64encode(b"\x89PNG\r\n\x1a\nscripted").decode()
+_JSON_LIST: Final = TypeAdapter(list[JsonValue])
+
+_NULL: Final[Mapping[str, JsonValue]] = {"type": "null"}
+_STRING: Final[Mapping[str, JsonValue]] = {"type": "string"}
+_INTEGER: Final[Mapping[str, JsonValue]] = {"type": "integer"}
+_NULLABLE_STRING: Final[JsonValue] = {"anyOf": [{"type": "string", "nullable": True}]}
+_NULLABLE_NULL: Final[JsonValue] = {"anyOf": [{"type": "null", "nullable": True}]}
+
+_UNIONS: Final[tuple[tuple[str, JsonValue, JsonValue], ...]] = (
+    ("null-null-string", {"anyOf": [_NULL, _NULL, _STRING]}, _NULLABLE_STRING),
+    ("null-only", {"anyOf": [_NULL]}, _NULLABLE_NULL),
+    ("two-nulls-only", {"anyOf": [_NULL, _NULL]}, _NULLABLE_NULL),
+    ("string-null", {"anyOf": [_STRING, _NULL]}, _NULLABLE_STRING),
+    (
+        "integer-null-null-string",
+        {"anyOf": [_INTEGER, _NULL, _NULL, _STRING]},
+        {"anyOf": [{"type": "integer", "nullable": True}, {"type": "string", "nullable": True}]},
+    ),
+    ("null-empty", {"anyOf": [_NULL, {}]}, {"anyOf": [{"type": "object", "nullable": True}]}),
+    ("empty-only", {"anyOf": [{}]}, {"anyOf": [{"type": "object"}]}),
+    ("empty-list", {"anyOf": []}, {"anyOf": []}),
+    ("object-anyof", {"anyOf": _STRING}, {"anyOf": _STRING}),
+    ("null-anyof", {"anyOf": None}, {"anyOf": None}),
+    ("five-hundred-nulls-then-string", {"anyOf": [*([_NULL] * 500), _STRING]}, _NULLABLE_STRING),
+    ("string-null-type-array", {"type": ["string", "null"]}, _NULLABLE_STRING),
+    (
+        "null-inside-a-union-branch",
+        {"anyOf": [{"type": "object", "properties": {"x": {"anyOf": [_INTEGER, _NULL]}}}, _NULL]},
+        {
+            "anyOf": [
+                {
+                    "type": "object",
+                    "properties": {"x": {"anyOf": [{"type": "integer", "nullable": True}]}},
+                    "nullable": True,
+                }
+            ]
+        },
+    ),
+)
+_FANOUT_UNIONS: Final = tuple(case for case in _UNIONS if case[0] in {"null-null-string", "null-only", "object-anyof"})
+
+
+def _text_reply() -> Mapping[str, JsonValue]:
+    return {
+        "candidates": [{"content": {"role": "model", "parts": [{"text": "OK"}]}, "finishReason": "STOP"}],
+        "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 1, "totalTokenCount": 10},
+        "modelVersion": _BACKEND,
+    }
+
+
+def _image_reply() -> Mapping[str, JsonValue]:
+    return {
+        "candidates": [
+            {"content": {"role": "model", "parts": [{"inlineData": {"mimeType": "image/png", "data": _PNG}}]}}
+        ],
+        "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 1, "totalTokenCount": 10},
+    }
+
+
+def _rejection(message: str) -> Reply:
+    error: Final = {"code": 400, "message": message, "status": "INVALID_ARGUMENT"}
+    return Reply(status=400, body=json.dumps({"error": error}).encode())
+
+
+def _google(request: Request) -> Reply:
+    if is_gcs_upload(request):
+        return gcs_peer(request)
+    if _FAIL_MARKER.encode() in request.body:
+        return _rejection(f"{_FAIL_MARKER}: the scripted tool schema was refused")
+    if _STRING_ANYOF.search(request.body):
+        return _rejection("Invalid value at 'tools[0].function_declarations[0].parameters' (anyOf), not a list")
+    if _SCALAR_BRANCH.search(request.body):
+        return _rejection(
+            "Invalid value at 'tools[0].function_declarations[0].parameters.properties[0].value.any_of[0]'"
+        )
+    path: Final = request.target.split("?")[0]
+    if path.endswith(":countTokens"):
+        return Reply(body=json.dumps({"totalTokens": _COUNT}).encode())
+    if path.endswith(":streamGenerateContent"):
+        return Reply(content_type="text/event-stream", chunks=(f"data: {json.dumps(_text_reply())}\n\n".encode(),))
+    if path.endswith(f"{_IMAGE_BACKEND}:generateContent"):
+        return Reply(body=json.dumps(_image_reply()).encode())
+    if path.endswith(":generateContent"):
+        return Reply(body=json.dumps(_text_reply()).encode())
+    return Reply(status=404, body=json.dumps({"error": {"code": 404, "message": request.target}}).encode())
+
+
+def _deployment(gateway: Gateway, scenario: Scenario, wire: Wire, provider: Provider, backend: str = _BACKEND) -> str:
+    if provider == "gemini":
+        return scenario.model(model=f"gemini/{backend}", api_base=wire.url, api_key=_GEMINI_KEY)
+    return scenario.model(
+        model=f"vertex_ai/{backend}",
+        api_base=wire.url,
+        api_key=None,
+        vertex_project=_PROJECT,
+        vertex_location=_LOCATION,
+        vertex_credentials=service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")),
+    )
+
+
+def _parameters(value: JsonValue) -> dict[str, JsonValue]:
+    return {"type": "object", "properties": {"value": value}, "required": ["value"]}
+
+
+def _declaration(description: str, value: JsonValue) -> dict[str, JsonValue]:
+    return {"name": "record", "description": description, "parameters": _parameters(value)}
+
+
+def _tool(endpoint: Endpoint, description: str, parameters: JsonValue) -> dict[str, JsonValue]:
+    if endpoint == "messages":
+        return {"name": "record", "description": description, "input_schema": parameters}
+    if endpoint == "responses":
+        return {"type": "function", "name": "record", "description": description, "parameters": parameters}
+    return {"type": "function", "function": {"name": "record", "description": description, "parameters": parameters}}
+
+
+def _request_body(
+    endpoint: Endpoint, model: str, description: str, parameters: JsonValue, stream: bool
+) -> tuple[str, dict[str, JsonValue]]:
+    prompt: Final = f"Record the value hello by calling the record tool. {description}"
+    tools: Final[list[JsonValue]] = [_tool(endpoint, description, parameters)]
+    streaming: Final[dict[str, JsonValue]] = {"stream": True} if stream else {}
+    if endpoint == "messages":
+        messages_body: Final[dict[str, JsonValue]] = {
+            "model": model,
+            "max_tokens": 16,
+            "messages": [{"role": "user", "content": prompt}],
+            "tools": tools,
+        }
+        return "/v1/messages", {**messages_body, **streaming}
+    if endpoint == "responses":
+        return "/v1/responses", {"model": model, "input": prompt, "tools": tools, **streaming}
+    chat_body: Final[dict[str, JsonValue]] = {
+        "model": model,
+        "max_tokens": 16,
+        "messages": [{"role": "user", "content": prompt}],
+        "tools": tools,
+    }
+    return "/v1/chat/completions", {**chat_body, **streaming}
+
+
+def _body(request: Request) -> dict[str, JsonValue]:
+    return JSON_OBJECT.validate_json(request.body)
+
+
+def _json(response: httpx.Response) -> dict[str, JsonValue]:
+    return JSON_OBJECT.validate_json(response.content)
+
+
+def _generate_requests(wire: Wire) -> tuple[Request, ...]:
+    return tuple(
+        request
+        for request in wire.drain()
+        if request.target.split("?")[0].endswith((":generateContent", ":streamGenerateContent"))
+    )
+
+
+def _sent_declarations(body: Mapping[str, JsonValue]) -> JsonValue:
+    tools: Final = _JSON_LIST.validate_python(body["tools"])
+    assert len(tools) == 1, tools
+    tool: Final = object_value(tools[0])
+    assert len(tool) == 1, tool
+    return next(iter(tool.values()))
+
+
+def _description() -> str:
+    return f"Record a value {uuid.uuid4().hex[:12]}"
+
+
+def _assert_caller_saw_ok(endpoint: Endpoint, response: httpx.Response, stream: bool) -> None:
+    assert response.status_code == 200, response.text
+    if stream:
+        assert "OK" in response.text, response.text
+        return
+    payload: Final = _json(response)
+    assert string_value(payload["id"]), payload
+    if endpoint == "chat":
+        choice: Final = object_value(_JSON_LIST.validate_python(payload["choices"])[0])
+        assert object_value(choice["message"])["content"] == "OK", payload
+        return
+    assert "OK" in response.text, payload
+
+
+@pytest.mark.parametrize("provider", ("vertex_ai", "gemini"))
+@pytest.mark.parametrize(("union", "expected"), [case[1:] for case in _UNIONS], ids=[case[0] for case in _UNIONS])
+def test_chat_tool_union_reaches_google_with_null_branches_folded_into_nullable(
+    gateway: Gateway, provider: Provider, union: JsonValue, expected: JsonValue
+) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, provider)
+        description: Final = _description()
+        path, body = _request_body("chat", model, description, _parameters(union), False)
+        response: Final = gateway.request("POST", path, body)
+        _assert_caller_saw_ok("chat", response, False)
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 1, [request.target for request in requests]
+        assert requests[0].target.split("?")[0].endswith(f"{_BACKEND}:generateContent"), requests[0].target
+        assert _sent_declarations(_body(requests[0])) == [_declaration(description, expected)]
+
+
+@pytest.mark.parametrize("stream", (False, True), ids=("whole", "stream"))
+@pytest.mark.parametrize("endpoint", ("chat", "messages", "responses"))
+@pytest.mark.parametrize("provider", ("vertex_ai", "gemini"))
+@pytest.mark.parametrize(
+    ("union", "expected"), [case[1:] for case in _FANOUT_UNIONS], ids=[case[0] for case in _FANOUT_UNIONS]
+)
+def test_every_unified_endpoint_sends_the_same_folded_union(
+    gateway: Gateway, provider: Provider, endpoint: Endpoint, stream: bool, union: JsonValue, expected: JsonValue
+) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, provider)
+        description: Final = _description()
+        path, body = _request_body(endpoint, model, description, _parameters(union), stream)
+        response: Final = gateway.request("POST", path, body)
+        _assert_caller_saw_ok(endpoint, response, stream)
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 1, [request.target for request in requests]
+        method: Final = "streamGenerateContent" if stream else "generateContent"
+        assert requests[0].target.split("?")[0].endswith(f"{_BACKEND}:{method}"), requests[0].target
+        assert _sent_declarations(_body(requests[0])) == [_declaration(description, expected)]
+
+
+def test_nested_item_and_property_unions_are_folded_at_every_depth(gateway: Gateway) -> None:
+    parameters: Final[dict[str, JsonValue]] = {
+        "type": "object",
+        "properties": {
+            "values": {"type": "array", "items": {"anyOf": [_NULL, _NULL, _INTEGER]}},
+            "nested": {"type": "object", "properties": {"inner": {"anyOf": [_NULL, _NULL, _STRING]}}},
+        },
+        "required": ["values", "nested"],
+    }
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, "vertex_ai")
+        description: Final = _description()
+        path, body = _request_body("chat", model, description, parameters, False)
+        response: Final = gateway.request("POST", path, body)
+        _assert_caller_saw_ok("chat", response, False)
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 1, [request.target for request in requests]
+        assert _sent_declarations(_body(requests[0])) == [
+            {
+                "name": "record",
+                "description": description,
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "values": {"type": "array", "items": {"anyOf": [{"type": "integer", "nullable": True}]}},
+                        "nested": {"type": "object", "properties": {"inner": _NULLABLE_STRING}},
+                    },
+                    "required": ["values", "nested"],
+                },
+            }
+        ]
+
+
+def test_a_string_anyof_reaches_google_and_its_400_reaches_the_caller(gateway: Gateway) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, "gemini")
+        description: Final = _description()
+        path, body = _request_body("chat", model, description, _parameters({"anyOf": "string"}), False)
+        response: Final = gateway.request("POST", path, body)
+        assert response.status_code == 400, response.text
+        assert "not a list" in response.text, response.text
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 1, [request.target for request in requests]
+        assert _sent_declarations(_body(requests[0])) == [_declaration(description, {"anyOf": "string"})]
+
+
+@pytest.mark.parametrize("provider", ("vertex_ai", "gemini"))
+def test_a_google_rejection_of_a_folded_union_reaches_the_caller(gateway: Gateway, provider: Provider) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, provider)
+        description: Final = f"{_FAIL_MARKER} {uuid.uuid4().hex[:12]}"
+        path, body = _request_body("chat", model, description, _parameters({"anyOf": [_NULL, _NULL, _STRING]}), False)
+        response: Final = gateway.request("POST", path, body)
+        assert response.status_code == 400, response.text
+        assert _FAIL_MARKER in response.text, response.text
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 1, [request.target for request in requests]
+        assert _sent_declarations(_body(requests[0])) == [_declaration(description, _NULLABLE_STRING)]
+
+
+@pytest.mark.parametrize(
+    ("union", "sent"),
+    (({"anyOf": [True, _NULL]}, {"anyOf": [True]}), ({"anyOf": [_NULL, 5]}, {"anyOf": [5]})),
+    ids=("true-null", "null-int"),
+)
+@pytest.mark.parametrize("provider", ("vertex_ai", "gemini"))
+def test_a_union_branch_that_is_not_an_object_reaches_google_and_its_400_reaches_the_caller(
+    gateway: Gateway, provider: Provider, union: JsonValue, sent: JsonValue
+) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, provider)
+        rejected_description: Final = _description()
+        path, broken = _request_body("chat", model, rejected_description, _parameters(union), False)
+        failed: Final = gateway.request("POST", path, broken)
+        assert failed.status_code == 400, failed.text
+        assert "any_of[0]" in failed.text, failed.text
+        rejected: Final = _generate_requests(wire)
+        assert len(rejected) == 1, [request.target for request in rejected]
+        assert _sent_declarations(_body(rejected[0])) == [_declaration(rejected_description, sent)]
+        description: Final = _description()
+        _, valid = _request_body("chat", model, description, _parameters({"anyOf": [_NULL, _NULL, _STRING]}), False)
+        recovered: Final = gateway.request("POST", path, valid)
+        _assert_caller_saw_ok("chat", recovered, False)
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 1, [request.target for request in requests]
+        assert _sent_declarations(_body(requests[0])) == [_declaration(description, _NULLABLE_STRING)]
+
+
+def test_a_repeated_identical_request_sends_the_same_folded_union_each_time(gateway: Gateway) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, "vertex_ai")
+        description: Final = _description()
+        path, body = _request_body("chat", model, description, _parameters({"anyOf": [_NULL, _NULL, _STRING]}), False)
+        first: Final = gateway.request("POST", path, body)
+        second: Final = gateway.request("POST", path, body)
+        _assert_caller_saw_ok("chat", first, False)
+        _assert_caller_saw_ok("chat", second, False)
+        assert _json(first)["id"] != _json(second)["id"]
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 2, [request.target for request in requests]
+        assert [_sent_declarations(_body(request)) for request in requests] == [
+            [_declaration(description, _NULLABLE_STRING)]
+        ] * 2
+
+
+def test_openai_sdk_chat_sends_the_folded_union(gateway: Gateway) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, "gemini")
+        description: Final = _description()
+        client: Final = openai.OpenAI(
+            base_url=f"{str(gateway.client.base_url).rstrip('/')}/v1", api_key=gateway.key, max_retries=0
+        )
+        completion: Final = client.chat.completions.create(
+            model=model,
+            max_tokens=16,
+            messages=[{"role": "user", "content": "Record the value hello by calling the record tool."}],
+            tools=[_tool("chat", description, _parameters({"anyOf": [_NULL, _NULL, _STRING]}))],  # pyright: ignore[reportArgumentType]  # SDK TypedDicts reject a JsonValue mapping
+        )
+        assert completion.choices[0].message.content == "OK"
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 1, [request.target for request in requests]
+        assert _sent_declarations(_body(requests[0])) == [_declaration(description, _NULLABLE_STRING)]
+
+
+async def _async_openai_stream_text(gateway: Gateway, model: str, description: str) -> str:
+    client: Final = openai.AsyncOpenAI(
+        base_url=f"{str(gateway.client.base_url).rstrip('/')}/v1", api_key=gateway.key, max_retries=0
+    )
+    stream: Final = await client.chat.completions.create(
+        model=model,
+        max_tokens=16,
+        stream=True,
+        messages=[{"role": "user", "content": "Record the value hello by calling the record tool."}],
+        tools=[_tool("chat", description, _parameters({"anyOf": [_NULL]}))],  # pyright: ignore[reportArgumentType]  # SDK TypedDicts reject a JsonValue mapping
+    )
+    return "".join([chunk.choices[0].delta.content or "" async for chunk in stream if chunk.choices])
+
+
+def test_async_openai_sdk_stream_sends_the_null_only_union_as_one_nullable_null(gateway: Gateway) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, "vertex_ai")
+        description: Final = _description()
+        assert asyncio.run(_async_openai_stream_text(gateway, model, description)) == "OK"
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 1, [request.target for request in requests]
+        assert requests[0].target.split("?")[0].endswith(":streamGenerateContent"), requests[0].target
+        assert _sent_declarations(_body(requests[0])) == [_declaration(description, _NULLABLE_NULL)]
+
+
+def test_anthropic_sdk_messages_sends_the_folded_union(gateway: Gateway) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, "vertex_ai")
+        description: Final = _description()
+        client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
+        message: Final = client.messages.create(
+            model=model,
+            max_tokens=16,
+            messages=[{"role": "user", "content": "Record the value hello by calling the record tool."}],
+            tools=[_tool("messages", description, _parameters({"anyOf": [_NULL, _NULL]}))],  # pyright: ignore[reportArgumentType]  # SDK TypedDicts reject a JsonValue mapping
+        )
+        assert [block.type for block in message.content] == ["text"]
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 1, [request.target for request in requests]
+        assert _sent_declarations(_body(requests[0])) == [_declaration(description, _NULLABLE_NULL)]
+
+
+@pytest.mark.parametrize(
+    ("union", "expected"), [case[1:] for case in _FANOUT_UNIONS], ids=[case[0] for case in _FANOUT_UNIONS]
+)
+def test_vertex_count_tokens_sends_the_folded_union(gateway: Gateway, union: JsonValue, expected: JsonValue) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, "vertex_ai")
+        description: Final = _description()
+        response: Final = gateway.request(
+            "POST",
+            "/v1/messages/count_tokens",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": "Record the value hello by calling the record tool."}],
+                "tools": [_tool("messages", description, _parameters(union))],
+            },
+        )
+        assert response.status_code == 200, response.text
+        assert _json(response)["input_tokens"] == _COUNT, response.text
+        requests: Final = tuple(request for request in wire.drain() if request.target.endswith(":countTokens"))
+        assert len(requests) == 1, [request.target for request in requests]
+        assert _sent_declarations(_body(requests[0])) == [_declaration(description, expected)]
+
+
+@pytest.mark.parametrize(
+    ("union", "expected"), [case[1:] for case in _FANOUT_UNIONS], ids=[case[0] for case in _FANOUT_UNIONS]
+)
+def test_gemini_token_counter_sends_the_folded_union(gateway: Gateway, union: JsonValue, expected: JsonValue) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, "gemini")
+        description: Final = _description()
+        response: Final = gateway.request(
+            "POST",
+            "/utils/token_counter",
+            {
+                "model": model,
+                "contents": [
+                    {"role": "user", "parts": [{"text": "Record the value hello by calling the record tool."}]}
+                ],
+                "tools": [_tool("chat", description, _parameters(union))],
+            },
+            params={"call_endpoint": "true"},
+        )
+        assert response.status_code == 200, response.text
+        assert _json(response)["total_tokens"] == _COUNT, response.text
+        requests: Final = tuple(request for request in wire.drain() if ":countTokens" in request.target)
+        assert len(requests) == 1, [request.target for request in requests]
+        sent: Final = object_value(_body(requests[0])["generateContentRequest"])
+        assert _sent_declarations(sent) == [_declaration(description, expected)]
+
+
+def test_response_format_on_a_model_without_json_schema_support_folds_the_union(gateway: Gateway) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, "gemini", _NO_JSON_SCHEMA_BACKEND)
+        response: Final = gateway.request(
+            "POST",
+            "/v1/chat/completions",
+            {
+                "model": model,
+                "messages": [{"role": "user", "content": "Return the value as JSON."}],
+                "response_format": {
+                    "type": "json_schema",
+                    "json_schema": {"name": "record", "schema": _parameters({"anyOf": [_NULL, _NULL, _STRING]})},
+                },
+            },
+        )
+        _assert_caller_saw_ok("chat", response, False)
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 1, [request.target for request in requests]
+        generation_config: Final = object_value(_body(requests[0])["generationConfig"])
+        assert generation_config["response_mime_type"] == "application/json", generation_config
+        assert generation_config["response_schema"] == {**_parameters(_NULLABLE_STRING), "propertyOrdering": ["value"]}
+
+
+def test_native_generate_content_response_schema_folds_the_union(gateway: Gateway) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, "gemini", _NO_JSON_SCHEMA_BACKEND)
+        response: Final = gateway.request(
+            "POST",
+            f"/v1beta/models/{model}:generateContent",
+            {
+                "contents": [{"role": "user", "parts": [{"text": "Return the value as JSON."}]}],
+                "generationConfig": {
+                    "responseMimeType": "application/json",
+                    "responseSchema": _parameters({"anyOf": [_NULL, _NULL, _STRING]}),
+                },
+            },
+        )
+        assert response.status_code == 200, response.text
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 1, [request.target for request in requests]
+        generation_config: Final = object_value(_body(requests[0])["generationConfig"])
+        assert generation_config["responseSchema"] == {**_parameters(_NULLABLE_STRING), "propertyOrdering": ["value"]}
+
+
+@pytest.mark.parametrize(
+    ("union", "expected"), [case[1:] for case in _FANOUT_UNIONS], ids=[case[0] for case in _FANOUT_UNIONS]
+)
+def test_image_generation_tools_send_the_folded_union(gateway: Gateway, union: JsonValue, expected: JsonValue) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = _deployment(gateway, scenario, wire, "gemini", _IMAGE_BACKEND)
+        description: Final = _description()
+        response: Final = gateway.request(
+            "POST",
+            "/v1/images/generations",
+            {
+                "model": model,
+                "prompt": "A small red square on a white background",
+                "tools": [_tool("chat", description, _parameters(union))],
+            },
+        )
+        assert response.status_code == 200, response.text
+        assert len(_JSON_LIST.validate_python(_json(response)["data"])) == 1, response.text
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == 1, [request.target for request in requests]
+        assert _sent_declarations(_body(requests[0])) == [_declaration(description, expected)]
+
+
+def test_vertex_batch_row_is_stored_with_the_folded_union(gateway: Gateway) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        model: Final = vertex_model(gateway, scenario, wire.url, _BACKEND)
+        description: Final = _description()
+        row: Final[dict[str, JsonValue]] = {
+            "custom_id": "anyof-row",
+            "method": "POST",
+            "url": "/v1/chat/completions",
+            "body": {
+                "model": model,
+                "messages": [{"role": "user", "content": "Record the value hello by calling the record tool."}],
+                "tools": [_tool("chat", description, _parameters({"anyOf": [_NULL, _NULL, _STRING]}))],
+            },
+        }
+        response: Final = upload(gateway, "model", model, jsonl(encoded(row)))
+        assert response.status_code == 200, response.text
+        uploads: Final = tuple(request for request in wire.drain() if is_gcs_upload(request))
+        assert len(uploads) == 1, [request.target for request in uploads]
+        stored: Final = tuple(
+            object_value(JSON_OBJECT.validate_json(line)["request"]) for line in uploads[0].body.splitlines() if line
+        )
+        assert len(stored) == 1, stored
+        assert _sent_declarations(stored[0]) == [_declaration(description, _NULLABLE_STRING)]
+
+
+async def _event_types_until_done(socket: websockets.ClientConnection) -> AsyncIterator[str]:
+    async for message in socket:
+        kind: Final = string_value(JSON_OBJECT.validate_json(message)["type"])
+        yield kind
+        if kind in {"response.done", "error"}:
+            return
+
+
+async def _realtime_session_with_tools(gateway: Gateway, model: str, tools: list[JsonValue]) -> tuple[str, ...]:
+    ws_base: Final = str(gateway.client.base_url).rstrip("/").replace("http://", "ws://")
+    headers: Final = {"Authorization": f"Bearer {gateway.key}"}
+    async with websockets.connect(f"{ws_base}/v1/realtime?model={model}", additional_headers=headers) as socket:
+        first: Final = JSON_OBJECT.validate_json(await socket.recv())
+        await socket.send(json.dumps({"type": "session.update", "session": {"tools": tools}}))
+        item: Final = {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "say the line"}]}
+        await socket.send(json.dumps({"type": "conversation.item.create", "item": item}))
+        events: Final = [kind async for kind in _event_types_until_done(socket)]
+        return (string_value(first["type"]), *events)
+
+
+def _realtime_types(gateway: Gateway, model: str, tools: list[JsonValue]) -> tuple[str, ...]:
+    async def bounded() -> tuple[str, ...]:
+        try:
+            return await asyncio.wait_for(_realtime_session_with_tools(gateway, model, tools), 30)
+        except TimeoutError:
+            return ("timeout",)
+
+    return asyncio.run(bounded())
+
+
+def _live_frames(gateway: Gateway, scenario_id: str) -> tuple[dict[str, JsonValue], ...]:
+    with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream:
+        observed: Final = tuple(
+            map(object_value, _JSON_LIST.validate_python(_json(upstream.get("/__observations"))["requests"]))
+        )
+    return tuple(
+        object_value(request["body"])
+        for request in observed
+        if request["api_key"] == scenario_id
+        and request["method"] == "WEBSOCKET_FRAME"
+        and request["path"] == GEMINI_LIVE_PATH
+    )
+
+
+@pytest.fixture(scope="module")
+def deferred_live(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]:
+    directory: Final = tmp_path_factory.mktemp("anyof-live")
+    with gateway_from_environment() as gateway:
+        with owned_proxy_process(gateway, directory, {"LITELLM_GEMINI_LIVE_DEFER_SETUP": "true"}, workers=2) as proxy:
+            yield proxy
+
+
+@pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
+def test_deferred_realtime_setup_carries_session_tools_with_the_folded_union(deferred_live: OwnedProxy) -> None:
+    gateway: Final = deferred_live.gateway
+    with gateway.scenario() as scenario:
+        handle: Final = live_scenario(scenario)
+        model: Final = live_deployment(scenario, handle.scenario_id, gateway.upstream_url)
+        description: Final = _description()
+        tools: Final[list[JsonValue]] = [_tool("chat", description, _parameters({"anyOf": [_NULL, _NULL, _STRING]}))]
+        types: Final = _realtime_types(gateway, model, tools)
+        assert types[0] == "session.created", types
+        assert types[-1] == "response.done", types
+        frames: Final = _live_frames(gateway, handle.scenario_id)
+        setups: Final = tuple(object_value(frame["setup"]) for frame in frames if "setup" in frame)
+        assert len(setups) == 1, [sorted(frame) for frame in frames]
+        assert _sent_declarations(setups[0]) == [_declaration(description, _NULLABLE_STRING)]
+
+
+_PROVIDERS: Final[tuple[Provider, ...]] = ("vertex_ai", "gemini")
+_ENDPOINTS: Final[tuple[Endpoint, ...]] = ("chat", "messages", "responses")
+_BURST: Final[tuple[tuple[Provider, Endpoint, bool, bool], ...]] = tuple(
+    (provider, endpoint, stream, index % 4 == 0)
+    for index, (provider, endpoint, stream, _) in enumerate(product(_PROVIDERS, _ENDPOINTS, (False, True), range(2)))
+)
+
+
+def test_a_concurrent_mixed_burst_folds_every_union_once_and_keeps_each_status(gateway: Gateway) -> None:
+    with wire_server(_google) as wire, gateway.scenario() as scenario:
+        models: Final = MappingProxyType(
+            {provider: _deployment(gateway, scenario, wire, provider) for provider in _PROVIDERS}
+        )
+        shots: Final = tuple(
+            (provider, endpoint, stream, f"{_FAIL_MARKER if refused else 'burst'} {uuid.uuid4().hex}")
+            for provider, endpoint, stream, refused in _BURST
+        )
+
+        def send(shot: tuple[Provider, Endpoint, bool, str]) -> httpx.Response:
+            provider, endpoint, stream, description = shot
+            path, body = _request_body(
+                endpoint, models[provider], description, _parameters({"anyOf": [_NULL, _NULL, _STRING]}), stream
+            )
+            return gateway.request("POST", path, body)
+
+        with ThreadPoolExecutor(max_workers=len(shots)) as pool:
+            responses: Final = tuple(pool.map(send, shots))
+        expected_statuses: Final = [400 if _FAIL_MARKER in shot[3] else 200 for shot in shots]
+        assert [response.status_code for response in responses] == expected_statuses, [
+            response.text[:120] for response in responses if response.status_code not in {200, 400}
+        ]
+        requests: Final = _generate_requests(wire)
+        assert len(requests) == len(shots), len(requests)
+        sent: Final = sorted((json.dumps(_sent_declarations(_body(request)), sort_keys=True) for request in requests))
+        assert sent == sorted(json.dumps([_declaration(shot[3], _NULLABLE_STRING)], sort_keys=True) for shot in shots)

tests/integration/providers/test_realtime_transcription_guardrail_session_update.py

@@ -8,10 +8,10 @@ while a client frame alone never flips a voice session into one. Every row runs
 which answers the injected and rewritten updates the way the vendors do.
 
 A blocked transcript on a transcription session reaches the client as a ``guardrail_violation`` error and
-nothing else: the ``response.cancel``, the voiced block prompt and the ``response.create`` a voice session gets
-never go to a backend that cannot speak, while ``on_violation: end_session`` and ``end_session_after_n_fails``
-still close the session. A guardrail that raises anything but a block closes the session the way it does on a
-voice session.
+nothing else: the ``response.cancel``, the blocked item's deletion, the voiced block prompt and the
+``response.create`` a voice session gets never go to a backend that cannot speak, while
+``on_violation: end_session`` and ``end_session_after_n_fails`` still close the session. A guardrail that raises
+anything but a block closes the session the way it does on a voice session.
 """
 
 from __future__ import annotations
@@ -51,6 +51,7 @@ from tests.integration._support.database import read_rows
 from tests.integration._support.process import (
     OwnedProxy,
     UpstreamSlot,
+    graceful_stop_seconds,
     owned_proxy_process,
     owned_upstream,
     stop_root_process,
@@ -60,7 +61,7 @@ from tests.integration.cost_calculation.cost_tracking_case import RealtimeRespon
 
 RecordProperty = Callable[[str, object], None]
 
-pytestmark: Final = pytest.mark.timeout(180)
+pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120)
 
 WORKERS: Final = 2
 BURST: Final = 20
@@ -83,7 +84,7 @@ CREATED_TYPES: Final = frozenset({"session.created", "transcription_session.crea
 TRANSCRIPT_COMPLETED: Final = "conversation.item.input_audio_transcription.completed"
 RESPONSE_DONE: Final = "response.done"
 GUARDRAIL_VIOLATION: Final = ("guardrail_violation", "content_policy_violation")
-MISSING_TURN_DETECTION_TYPE: Final = ("invalid_request_error", "missing_required_parameter")
+SESSION_TYPE_MISMATCH: Final = ("invalid_request_error", "invalid_parameter")
 INVALID_SESSION_VALUE: Final = ("invalid_request_error", "invalid_value")
 FIVE_KB: Final = "x" * 5120
 UNAUTHENTICATED_STATUS: Final = 403
@@ -104,10 +105,9 @@ BETA_TRANSCRIPTION_UPDATE: Final[dict[str, JsonValue]] = {
     "input_audio_transcription": {"model": TRANSCRIBE_MODEL},
 }
 VOICE_UPDATE: Final[dict[str, JsonValue]] = {"type": "realtime", "instructions": "answer briefly"}
-VOICE_UPDATE_FORWARDED: Final[dict[str, JsonValue]] = {
-    "type": "realtime",
-    "instructions": "answer briefly",
-    "audio": {"input": {"turn_detection": {"create_response": False}}},
+VOICE_UPDATE_GATED: Final[dict[str, JsonValue]] = {
+    **VOICE_UPDATE,
+    "audio": {"input": {"turn_detection": {"type": "server_vad", "create_response": False}}},
 }
 RE_ENABLE_UPDATE: Final[dict[str, JsonValue]] = {
     "type": "realtime",
@@ -123,14 +123,32 @@ CLIENT_DECLARED_TRANSCRIPTION: Final[dict[str, JsonValue]] = {
 }
 CLIENT_DECLARED_TRANSCRIPTION_FORWARDED: Final[dict[str, JsonValue]] = {
     "type": "transcription",
-    "audio": {"input": {"turn_detection": {"create_response": False}}},
+    "audio": {"input": {"turn_detection": {"type": "server_vad", "create_response": False}}},
 }
-TURN_DETECTION_TYPE_MISSING: Final = "Missing required parameter: 'session.audio.input.turn_detection.type'."
+REALTIME_UPDATE_REFUSED: Final = "Passing a transcription session update to a realtime session is not allowed."
 GA_INJECTED_UPDATE: Final[dict[str, JsonValue]] = {
     "type": "realtime",
     "audio": {"input": {"turn_detection": {"type": "server_vad", "create_response": False}}},
 }
 BETA_INJECTED_UPDATE: Final[dict[str, JsonValue]] = {"turn_detection": {"type": "server_vad", "create_response": False}}
+XAI_VOICE_MODEL: Final = "grok-voice"
+GEMINI_LIVE_MODEL: Final = "gemini-3.8-live"
+VERTEX_LIVE_LOCATION: Final = "us-central1"
+UNTYPED_GATE: Final[dict[str, JsonValue]] = {"create_response": False}
+SEMANTIC_VAD: Final[dict[str, JsonValue]] = {"type": "semantic_vad", "eagerness": "low"}
+TUNED_SERVER_VAD: Final[dict[str, JsonValue]] = {
+    "type": "server_vad",
+    "prefix_padding_ms": 300,
+    "silence_duration_ms": 500,
+}
+GEMINI_TUNED_DETECTION: Final[dict[str, JsonValue]] = {
+    "disabled": True,
+    "prefixPaddingMs": 300,
+    "silenceDurationMs": 500,
+}
+CLIENT_CANCEL_ID: Final = "client_cancel_1"
+CANCEL_NOT_ACTIVE: Final = ("invalid_request_error", "response_cancel_not_active")
+AUTO_ANSWER_PREFIX: Final = "resp_auto_"
 COMMIT: Final[dict[str, JsonValue]] = {"type": "input_audio_buffer.commit"}
 APPEND: Final[dict[str, JsonValue]] = {"type": "input_audio_buffer.append", "audio": PCM_200MS_24KHZ}
 PUSH_TO_TALK: Final = "PUSH_TO_TALK"
@@ -172,7 +190,13 @@ GUARDRAIL_END_CLOSE: Final = 1000
 PROXY_FAILURE_CLOSE: Final = 1011
 PROXY_FAILURE_REASON: Final = "proxy failed while relaying the upstream websocket"
 TOOL_OUTPUT_BLOCKED: Final = json.dumps({"error": "Tool output blocked by content policy"})
-VOICE_BLOCK_FRAMES: Final = ("response.cancel", "conversation.item.create", "response.create")
+VOICE_BLOCK_FRAMES: Final = (
+    "response.cancel",
+    "conversation.item.delete",
+    "conversation.item.create",
+    "response.create",
+)
+XAI_VOICE_BLOCK_FRAMES: Final = ("response.cancel", "conversation.item.create", "response.create")
 CREATED_SECONDS: Final = 60.0
 STEP_SECONDS: Final = 15.0
 SDK_SECONDS: Final = 45.0
@@ -318,6 +342,30 @@ def _voice_scenario(*transcripts: JsonValue) -> RealtimeResponse:
     return RealtimeResponse(content_type="application/x-realtime", events=tuple(_voice_turns(transcripts)))
 
 
+def _xai_voice_scenario(transcript: str) -> RealtimeResponse:
+    return RealtimeResponse(
+        content_type="application/x-realtime", events=tuple(_voice_turns((transcript,))), dialect="xai"
+    )
+
+
+def _voice_scenario_created_after_the_first_frame(transcript: str) -> RealtimeResponse:
+    return RealtimeResponse(
+        content_type="application/x-realtime", events=tuple(_voice_turns((transcript,))), created_after_first_frame=True
+    )
+
+
+def _live_scenario() -> RealtimeResponse:
+    return RealtimeResponse(content_type="application/x-realtime", events=())
+
+
+def _realtime_update(turn_detection: JsonValue) -> dict[str, JsonValue]:
+    return {"type": "realtime", "audio": {"input": {"turn_detection": turn_detection}}}
+
+
+def _without_auto_response(turn_detection: dict[str, JsonValue]) -> dict[str, JsonValue]:
+    return {**turn_detection, "create_response": False}
+
+
 def _muse_scenario(transcript: str) -> RealtimeResponse:
     return RealtimeResponse(
         content_type="application/x-realtime",
@@ -358,8 +406,41 @@ def _with_transcription_model(update: dict[str, JsonValue], model: str) -> dict[
     return {**update, "audio": {"input": {**audio_input, "transcription": {"model": model}}}}
 
 
-def _muse_deployment(scenario: Scenario, scenario_id: str, api_base: str) -> str:
-    return scenario.model(model=f"meta/{MUSE_MODEL}", api_key=scenario_id, api_base=api_base)
+def _muse_deployment(scenario: Scenario, scenario_id: str, api_base: str, *, drop_params: bool) -> str:
+    return scenario.model(model=f"meta/{MUSE_MODEL}", api_key=scenario_id, api_base=api_base, drop_params=drop_params)
+
+
+def _gemini_studio_deployment(scenario: Scenario, scenario_id: str, upstream_url: str) -> str:
+    return scenario.model(
+        model=f"gemini/{GEMINI_LIVE_MODEL}",
+        api_key=scenario_id,
+        api_base=upstream_url.rstrip("/"),
+        model_info={"mode": "realtime"},
+    )
+
+
+def _vertex_credentials(upstream_url: str) -> str:
+    return json.dumps(
+        {
+            "type": "external_account",
+            "audience": "synthetic-vertex-audience",
+            "subject_token_type": "urn:ietf:params:oauth:token-type:jwt",
+            "token_url": f"{upstream_url}/_oauth/token",
+            "credential_source": {"url": f"{upstream_url}/health"},
+        }
+    )
+
+
+def _vertex_live_deployment(scenario: Scenario, project: str, upstream_url: str) -> str:
+    upstream: Final = upstream_url.rstrip("/")
+    return scenario.model(
+        model=f"vertex_ai/{GEMINI_LIVE_MODEL}",
+        vertex_project=project,
+        vertex_credentials=_vertex_credentials(upstream),
+        vertex_location=VERTEX_LIVE_LOCATION,
+        api_base=upstream,
+        model_info={"mode": "realtime"},
+    )
 
 
 def _named_deployment_id(creator: Gateway, scenario: Scenario, name: str, model: str, scenario_id: str) -> str:
@@ -549,6 +630,55 @@ def _transcribe(
     return asyncio.run(_session(ws_base, path, query, key, frames, until=until, headers=headers))
 
 
+async def _received_after_sending(
+    socket: ClientConnection, frames: tuple[dict[str, JsonValue], ...], until: str
+) -> AsyncIterator[dict[str, JsonValue]]:
+    try:
+        for frame in frames:
+            await socket.send(json.dumps(frame))
+        async for event in _until(socket, until, STEP_SECONDS):
+            yield event
+    except ConnectionClosed as closed:
+        yield {"type": "closed", "code": _close_code(closed)}
+
+
+def _is_proxy_answer(event: dict[str, JsonValue]) -> bool:
+    if event.get("type") != RESPONSE_DONE:
+        return False
+    return not string_value(object_value(event["response"])["id"]).startswith(AUTO_ANSWER_PREFIX)
+
+
+async def _until_the_proxy_answer(
+    socket: ClientConnection, frames: tuple[dict[str, JsonValue], ...]
+) -> AsyncIterator[dict[str, JsonValue]]:
+    try:
+        yield JSON_OBJECT.validate_json(await asyncio.wait_for(socket.recv(), CREATED_SECONDS))
+        for frame in frames:
+            await socket.send(json.dumps(frame))
+        deadline: Final = asyncio.get_running_loop().time() + STEP_SECONDS
+        while (event := await _next_event(socket, deadline)) is not None:
+            yield event
+            if _is_proxy_answer(event):
+                return
+        yield {"type": "timeout"}
+    except ConnectionClosed as closed:
+        yield {"type": "closed", "code": _close_code(closed)}
+
+
+async def _talked_until_the_proxy_answer(
+    ws_base: str, query: str, key: str, frames: tuple[dict[str, JsonValue], ...]
+) -> Session:
+    headers: Final = {"Authorization": f"Bearer {key}"}
+    async with websockets.connect(f"{ws_base}/v1/realtime?{query}", additional_headers=headers) as socket:
+        return Session(tuple([event async for event in _until_the_proxy_answer(socket, frames)]), None)
+
+
+async def _talked_at_connect(ws_base: str, query: str, key: str, frames: tuple[dict[str, JsonValue], ...]) -> Session:
+    headers: Final = {"Authorization": f"Bearer {key}"}
+    async with websockets.connect(f"{ws_base}/v1/realtime?{query}", additional_headers=headers) as socket:
+        return Session(tuple([event async for event in _received_after_sending(socket, frames, RESPONSE_DONE)]), None)
+
+
 def _talk(
     ws_base: str,
     query: str,
@@ -617,10 +747,43 @@ def _session_updates(observed: tuple[dict[str, JsonValue], ...]) -> tuple[JsonVa
     return tuple(frame.get("session") for frame in _sent(observed) if frame.get("type") == "session.update")
 
 
+def _assert_voice_updates(observed: tuple[dict[str, JsonValue], ...], *later: dict[str, JsonValue]) -> None:
+    updates: Final = _session_updates(observed)
+    assert updates[0] == GA_INJECTED_UPDATE, updates
+    assert updates[1] == VOICE_UPDATE_GATED, updates
+    assert updates[2:] == later, updates
+
+
 def _query(upgrade: dict[str, JsonValue]) -> JsonValue:
     return upgrade["query"]
 
 
+def _declared_turn_detection(update: JsonValue) -> JsonValue:
+    session: Final = object_value(update)
+    audio_input: Final = object_value(object_value(session.get("audio", {})).get("input", {}))
+    return audio_input["turn_detection"] if "turn_detection" in audio_input else session.get("turn_detection")
+
+
+def _upstream_turn_detections(observed: tuple[dict[str, JsonValue], ...]) -> tuple[JsonValue, ...]:
+    return tuple(_declared_turn_detection(update) for update in _session_updates(observed))
+
+
+def _response_ids(session: Session) -> tuple[str, ...]:
+    done: Final = tuple(object_value(event["response"]) for event in session.events if event["type"] == RESPONSE_DONE)
+    return tuple(string_value(response["id"]) for response in done)
+
+
+def _error_event_ids(session: Session) -> tuple[JsonValue, ...]:
+    errors: Final = tuple(object_value(event["error"]) for event in session.events if event["type"] == "error")
+    return tuple(error.get("event_id") for error in errors)
+
+
+def _setup_detection(observed: tuple[dict[str, JsonValue], ...]) -> JsonValue:
+    setups: Final = tuple(object_value(frame["setup"]) for frame in _sent(observed) if "setup" in frame)
+    assert len(setups) == 1, _sent(observed)
+    return object_value(setups[0].get("realtimeInputConfig", {})).get("automaticActivityDetection")
+
+
 def _spend_rows(key: str, count: int) -> list[dict[str, JsonValue]]:
     return eventually(
         lambda: read_rows(SPEND_SQL, (sha256(key.encode()).hexdigest(),)),
@@ -704,6 +867,7 @@ def _write_config(directory: Path, ca_bundle: Path) -> Path:
                 "litellm_settings": {
                     "ssl_verify": str(ca_bundle),
                     "enable_redis_auth_cache": True,
+                    "gemini_live_defer_setup": True,
                     "cache": True,
                     "cache_params": {"type": "redis", "host": "os.environ/REDIS_HOST", "port": "os.environ/REDIS_PORT"},
                 },
@@ -918,19 +1082,18 @@ def test_voice_session_keeps_the_injected_update_and_the_clean_transcript_flows(
         assert session.types == (
             "session.created",
             "session.updated",
-            "error",
+            "session.updated",
             TRANSCRIPT_COMPLETED,
             RESPONSE_DONE,
         ), session
-        assert session.errors == (MISSING_TURN_DETECTION_TYPE,), session
-        assert session.error_messages == (TURN_DETECTION_TYPE_MISSING,), session
+        assert session.errors == (), session
         assert _sent_types(observed) == (
             "session.update",
             "session.update",
             "input_audio_buffer.commit",
             "response.create",
         ), _sent(observed)
-        assert _session_updates(observed) == (GA_INJECTED_UPDATE, VOICE_UPDATE_FORWARDED), _session_updates(observed)
+        _assert_voice_updates(observed)
         rows: Final = _spend_rows(key, 1)
         assert rows[0]["call_type"] == "_arealtime", rows
 
@@ -945,17 +1108,15 @@ def test_voice_session_blocked_transcript_is_refused_through_the_backend(guardra
         )
         observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id)
         assert session.transcripts == (BLOCKED_TRANSCRIPT,), session
-        assert session.errors == (MISSING_TURN_DETECTION_TYPE, GUARDRAIL_VIOLATION), session
+        assert session.errors == (GUARDRAIL_VIOLATION,), session
         assert session.types[-1] == RESPONSE_DONE, session
         assert _sent_types(observed) == (
             "session.update",
             "session.update",
             "input_audio_buffer.commit",
-            "response.cancel",
-            "conversation.item.create",
-            "response.create",
+            *VOICE_BLOCK_FRAMES,
         ), _sent(observed)
-        assert _session_updates(observed) == (GA_INJECTED_UPDATE, VOICE_UPDATE_FORWARDED), _session_updates(observed)
+        _assert_voice_updates(observed)
 
 
 def test_voice_session_cannot_re_enable_auto_response_with_a_later_update(guardrail_proxy: OwnedProxy) -> None:
@@ -967,12 +1128,8 @@ def test_voice_session_cannot_re_enable_auto_response_with_a_later_update(guardr
         session: Final = _talk(_ws_base(_owned_url(guardrail_proxy)), f"model={model}", key, frames)
         observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id)
         assert session.types[-1] == RESPONSE_DONE, session
-        assert session.errors == (MISSING_TURN_DETECTION_TYPE,), session
-        assert _session_updates(observed) == (
-            GA_INJECTED_UPDATE,
-            VOICE_UPDATE_FORWARDED,
-            RE_ENABLE_FORWARDED,
-        ), _session_updates(observed)
+        assert session.errors == (), session
+        _assert_voice_updates(observed, RE_ENABLE_FORWARDED)
 
 
 def test_client_declared_transcription_type_does_not_bypass_the_voice_guardrail(guardrail_proxy: OwnedProxy) -> None:
@@ -984,16 +1141,14 @@ def test_client_declared_transcription_type_does_not_bypass_the_voice_guardrail(
         session: Final = _talk(_ws_base(_owned_url(guardrail_proxy)), f"model={model}", key, frames, seconds=20)
         observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id)
         assert session.transcripts == (BLOCKED_TRANSCRIPT,), session
-        assert session.errors == (MISSING_TURN_DETECTION_TYPE, GUARDRAIL_VIOLATION), session
-        assert session.error_messages[0] == TURN_DETECTION_TYPE_MISSING, session
+        assert session.errors == (SESSION_TYPE_MISMATCH, GUARDRAIL_VIOLATION), session
+        assert session.error_messages[0] == REALTIME_UPDATE_REFUSED, session
         assert session.types[-1] == RESPONSE_DONE, session
         assert _sent_types(observed) == (
             "session.update",
             "session.update",
             "input_audio_buffer.commit",
-            "response.cancel",
-            "conversation.item.create",
-            "response.create",
+            *VOICE_BLOCK_FRAMES,
         ), _sent(observed)
         assert _session_updates(observed) == (
             GA_INJECTED_UPDATE,
@@ -1014,6 +1169,202 @@ def test_backend_session_created_typed_transcription_skips_the_injection_on_the_
         assert [_query(upgrade) for upgrade in _upgrades(observed)] == [[["model", TRANSCRIBE_MODEL]]]
 
 
+def test_voice_session_typed_turn_detection_keeps_its_type_and_tuning(guardrail_proxy: OwnedProxy) -> None:
+    with guardrail_proxy.gateway.scenario() as scenario:
+        handle: Final = _scripted(scenario, _voice_scenario(CLEAN_TRANSCRIPT))
+        key: Final = scenario.key()
+        model: Final = _openai_deployment(scenario, handle.scenario_id, model=VOICE_MODEL)
+        frames: Final = _voice_frames(_realtime_update(SEMANTIC_VAD))
+        session: Final = _talk(_ws_base(_owned_url(guardrail_proxy)), f"model={model}", key, frames)
+        observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id)
+        assert session.errors == (), session
+        assert session.types[-1] == RESPONSE_DONE, session
+        assert _session_updates(observed) == (
+            GA_INJECTED_UPDATE,
+            _realtime_update(_without_auto_response(SEMANTIC_VAD)),
+        ), _session_updates(observed)
+
+
+@dataclass(frozen=True, slots=True)
+class LaterUpdateCase:
+    headers: Mapping[str, str]
+    first: dict[str, JsonValue]
+    later: dict[str, JsonValue]
+    forwarded: tuple[dict[str, JsonValue], ...]
+
+
+LATER_UPDATE_CASES: Final = (
+    pytest.param(
+        LaterUpdateCase(
+            {},
+            VOICE_UPDATE,
+            {"type": "realtime", "instructions": "later"},
+            (GA_INJECTED_UPDATE, VOICE_UPDATE_GATED),
+        ),
+        id="ga",
+    ),
+    pytest.param(
+        LaterUpdateCase(
+            BETA_HEADERS,
+            {"turn_detection": {"type": "server_vad"}},
+            {"instructions": "later"},
+            (BETA_INJECTED_UPDATE, BETA_INJECTED_UPDATE),
+        ),
+        id="beta",
+    ),
+)
+
+
+@pytest.mark.parametrize("case", LATER_UPDATE_CASES)
+def test_later_update_without_turn_detection_is_forwarded_verbatim(
+    guardrail_proxy: OwnedProxy, case: LaterUpdateCase
+) -> None:
+    with guardrail_proxy.gateway.scenario() as scenario:
+        handle: Final = _scripted(scenario, _voice_scenario(CLEAN_TRANSCRIPT))
+        key: Final = scenario.key()
+        model: Final = _openai_deployment(scenario, handle.scenario_id, model=VOICE_MODEL)
+        session: Final = _drive(
+            _ws_base(_owned_url(guardrail_proxy)),
+            f"model={model}",
+            key,
+            (_step(_update_frame(case.first), _update_frame(case.later), COMMIT, until=RESPONSE_DONE),),
+            headers=case.headers,
+        )
+        observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id)
+        assert session.errors == (), session
+        assert session.types[-1] == RESPONSE_DONE, session
+        assert _session_updates(observed) == (*case.forwarded, case.later), _session_updates(observed)
+
+
+def test_session_update_sent_before_session_created_follows_the_gate(guardrail_proxy: OwnedProxy) -> None:
+    with guardrail_proxy.gateway.scenario() as scenario:
+        handle: Final = _scripted(scenario, _voice_scenario_created_after_the_first_frame(CLEAN_TRANSCRIPT))
+        key: Final = scenario.key()
+        model: Final = _openai_deployment(scenario, handle.scenario_id, model=VOICE_MODEL)
+        frames: Final = _voice_frames(_realtime_update(TUNED_SERVER_VAD))
+        session: Final = asyncio.run(
+            _talked_at_connect(_ws_base(_owned_url(guardrail_proxy)), f"model={model}", key, frames)
+        )
+        observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id)
+        assert session.types == (
+            "session.created",
+            "session.updated",
+            "session.updated",
+            TRANSCRIPT_COMPLETED,
+            RESPONSE_DONE,
+        ), session
+        assert session.errors == (), session
+        assert _session_updates(observed) == (
+            GA_INJECTED_UPDATE,
+            _realtime_update(_without_auto_response(TUNED_SERVER_VAD)),
+        ), _session_updates(observed)
+
+
+def test_client_response_cancel_error_still_reaches_the_client(guardrail_proxy: OwnedProxy) -> None:
+    with guardrail_proxy.gateway.scenario() as scenario:
+        handle: Final = _scripted(scenario, _voice_scenario(CLEAN_TRANSCRIPT))
+        key: Final = scenario.key()
+        model: Final = _openai_deployment(scenario, handle.scenario_id, model=VOICE_MODEL)
+        cancel: Final[dict[str, JsonValue]] = {"type": "response.cancel", "event_id": CLIENT_CANCEL_ID}
+        session: Final = _drive(
+            _ws_base(_owned_url(guardrail_proxy)), f"model={model}", key, (_step(cancel, until="error"),)
+        )
+        assert session.errors == (CANCEL_NOT_ACTIVE,), session
+        assert _error_event_ids(session) == (CLIENT_CANCEL_ID,), session
+
+
+def _xai_blocked_session(
+    guardrail_proxy: OwnedProxy, scenario: Scenario
+) -> tuple[Session, tuple[dict[str, JsonValue], ...]]:
+    handle: Final = _scripted(scenario, _xai_voice_scenario(BLOCKED_TRANSCRIPT))
+    key: Final = scenario.key()
+    model: Final = scenario.model(model=f"xai/{XAI_VOICE_MODEL}", api_key=handle.scenario_id)
+    frames: Final = _voice_frames(_realtime_update(TUNED_SERVER_VAD))
+    session: Final = asyncio.run(
+        _talked_until_the_proxy_answer(_ws_base(_owned_url(guardrail_proxy)), f"model={model}", key, frames)
+    )
+    return session, _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id)
+
+
+def test_xai_voice_session_gets_untyped_gates_and_no_unguarded_answer(guardrail_proxy: OwnedProxy) -> None:
+    with guardrail_proxy.gateway.scenario() as scenario:
+        session, observed = _xai_blocked_session(guardrail_proxy, scenario)
+        turn_detections: Final = _upstream_turn_detections(observed)
+        assert turn_detections == (UNTYPED_GATE, UNTYPED_GATE), turn_detections
+        assert session.transcripts == (BLOCKED_TRANSCRIPT,), session
+        assert GUARDRAIL_VIOLATION in session.errors, session
+        assert not any(response.startswith(AUTO_ANSWER_PREFIX) for response in _response_ids(session)), session
+
+
+def test_xai_voice_session_block_sends_xai_no_item_delete(guardrail_proxy: OwnedProxy) -> None:
+    with guardrail_proxy.gateway.scenario() as scenario:
+        session, observed = _xai_blocked_session(guardrail_proxy, scenario)
+        assert session.transcripts == (BLOCKED_TRANSCRIPT,), session
+        assert _sent_types(observed) == (
+            "session.update",
+            "session.update",
+            "input_audio_buffer.commit",
+            *XAI_VOICE_BLOCK_FRAMES,
+        ), _sent(observed)
+
+
+@pytest.mark.xfail(
+    strict=True,
+    raises=AssertionError,
+    reason=(
+        "xAI answers the guardrail's response.cancel with an error carrying its own event_id, so the proxy cannot "
+        "tell it apart from an error for a client event and forwards it, as the merge base does"
+    ),
+)
+def test_xai_guardrail_cancel_error_does_not_reach_the_client(guardrail_proxy: OwnedProxy) -> None:
+    with guardrail_proxy.gateway.scenario() as scenario:
+        session, _ = _xai_blocked_session(guardrail_proxy, scenario)
+        assert session.errors == (GUARDRAIL_VIOLATION,), session
+
+
+TUNED_VOICE_UPDATES: Final = (
+    pytest.param(_realtime_update(TUNED_SERVER_VAD), id="nested"),
+    pytest.param({"turn_detection": TUNED_SERVER_VAD}, id="flat"),
+)
+
+
+@pytest.mark.parametrize("update", TUNED_VOICE_UPDATES)
+def test_gemini_ai_studio_voice_session_with_vad_tuning_connects_under_the_guardrail(
+    guardrail_proxy: OwnedProxy, update: dict[str, JsonValue]
+) -> None:
+    with guardrail_proxy.gateway.scenario() as scenario:
+        handle: Final = _scripted(scenario, _live_scenario())
+        key: Final = scenario.key()
+        upstream_url: Final = guardrail_proxy.gateway.upstream_url
+        model: Final = _gemini_studio_deployment(scenario, handle.scenario_id, upstream_url)
+        session: Final = _drive(
+            _ws_base(_owned_url(guardrail_proxy)),
+            f"model={model}",
+            key,
+            (_step(_update_frame(update), until="session.updated"),),
+        )
+        observed: Final = _observed(upstream_url, handle.scenario_id)
+        assert session.types == ("session.created", "session.updated"), session
+        assert _setup_detection(observed) == {"disabled": True}, _sent(observed)
+
+
+def test_vertex_live_voice_session_keeps_vad_tuning_under_the_guardrail(guardrail_proxy: OwnedProxy) -> None:
+    with guardrail_proxy.gateway.scenario() as scenario:
+        handle: Final = _scripted(scenario, _live_scenario())
+        key: Final = scenario.key()
+        upstream_url: Final = guardrail_proxy.gateway.upstream_url
+        model: Final = _vertex_live_deployment(scenario, handle.scenario_id, upstream_url)
+        session: Final = _drive(
+            _ws_base(_owned_url(guardrail_proxy)),
+            f"model={model}",
+            key,
+            (_step(_update_frame(_realtime_update(TUNED_SERVER_VAD)), until="session.updated"),),
+        )
+        observed: Final = _observed(upstream_url, handle.scenario_id)
+        assert session.types == ("session.created", "session.updated"), session
+        assert _setup_detection(observed) == GEMINI_TUNED_DETECTION, _sent(observed)
+
+
 MUSE_PUSH_TO_TALK_FRAMES: Final[tuple[dict[str, JsonValue], ...]] = (
     {"binary_bytes": MUSE_PACKET_BYTES},
     {"binary_bytes": MUSE_PACKET_BYTES},
@@ -1041,10 +1392,11 @@ def _muse_session(
     turn_detection: JsonValue,
     *,
     until: str | None = None,
+    drop_params: bool = False,
 ) -> tuple[Session, tuple[dict[str, JsonValue], ...]]:
     handle: Final = _scripted(scenario, _muse_scenario(transcript), control_url=tls_upstream)
     key: Final = scenario.key()
-    model: Final = _muse_deployment(scenario, handle.scenario_id, tls_upstream)
+    model: Final = _muse_deployment(scenario, handle.scenario_id, tls_upstream, drop_params=drop_params)
     verdict: Final = TRANSCRIPT_COMPLETED if BLOCKED_WORD not in transcript else "error"
     session: Final = _talk(
         _ws_base(_owned_url(guardrail_proxy)),
@@ -1120,6 +1472,40 @@ def test_muse_server_vad_transcription_session_keeps_endpointing(
         ), _muse_audio_frames(observed)
 
 
+def test_muse_semantic_vad_under_drop_params_falls_back_to_endpointing(
+    guardrail_proxy: OwnedProxy, tls_upstream: str
+) -> None:
+    with guardrail_proxy.gateway.scenario() as scenario:
+        session, observed = _muse_session(
+            guardrail_proxy,
+            scenario,
+            tls_upstream,
+            CLEAN_TRANSCRIPT,
+            f"&{TRANSCRIPTION_QUERY}",
+            {"type": "semantic_vad"},
+            drop_params=True,
+        )
+        assert session.transcripts == (CLEAN_TRANSCRIPT,), session
+        assert session.errors == (), session
+        assert _muse_handshake(observed)["mode"] == ENDPOINTING, _muse_handshake(observed)
+
+
+def test_muse_semantic_vad_without_drop_params_is_refused(guardrail_proxy: OwnedProxy, tls_upstream: str) -> None:
+    with guardrail_proxy.gateway.scenario() as scenario:
+        session, observed = _muse_session(
+            guardrail_proxy,
+            scenario,
+            tls_upstream,
+            CLEAN_TRANSCRIPT,
+            "",
+            {"type": "semantic_vad"},
+            until=TRANSCRIPT_COMPLETED,
+        )
+        assert session.types[-1] == "closed", session
+        assert session.transcripts == (), session
+        assert _upgrades(observed) == (), observed
+
+
 @pytest.mark.xfail(
     strict=True,
     reason=(
@@ -1229,8 +1615,8 @@ def test_opt_in_guardrail_gates_a_voice_session_on_an_opted_out_key(guardrail_pr
         )
         observed: Final = _observed(guardrail_proxy.gateway.upstream_url, handle.scenario_id)
         assert session.transcripts == (BLOCKED_TRANSCRIPT,), session
-        assert session.errors == (MISSING_TURN_DETECTION_TYPE, GUARDRAIL_VIOLATION), session
-        assert _session_updates(observed) == (GA_INJECTED_UPDATE, VOICE_UPDATE_FORWARDED), _session_updates(observed)
+        assert session.errors == (GUARDRAIL_VIOLATION,), session
+        _assert_voice_updates(observed)
 
 
 def test_opt_in_guardrail_leaves_a_transcription_session_alone_on_an_opted_out_key(guardrail_proxy: OwnedProxy) -> None:
@@ -1717,12 +2103,11 @@ def test_voice_session_guardrail_raising_runtime_error_closes_with_the_same_prox
         assert session.types == (
             "session.created",
             "session.updated",
-            "error",
+            "session.updated",
             TRANSCRIPT_COMPLETED,
             "error",
             "closed",
         ), session
-        assert session.errors[0] == MISSING_TURN_DETECTION_TYPE, session
         _assert_closed_by_a_proxy_failure(session)
         assert _sent_types(observed) == ("session.update", "session.update", "input_audio_buffer.commit"), _sent(
             observed
@@ -1740,8 +2125,8 @@ def test_voice_session_guardrail_raising_value_error_is_voiced_through_the_backe
             (_step(VOICE_UPDATE_FRAME, COMMIT, until=RESPONSE_DONE),),
         )
         assert session.transcripts == (blocked,), session
-        assert session.errors == (MISSING_TURN_DETECTION_TYPE, GUARDRAIL_VIOLATION), session
-        assert session.error_messages[1] == f"{VALUE_ERROR_WORD} is not allowed", session
+        assert session.errors == (GUARDRAIL_VIOLATION,), session
+        assert session.error_messages == (f"{VALUE_ERROR_WORD} is not allowed",), session
         assert session.types[-1] == RESPONSE_DONE, session
         assert _sent_types(observed) == (
             "session.update",
@@ -1764,7 +2149,7 @@ def test_voice_session_second_violation_under_end_session_after_n_fails_closes_a
             (_step(VOICE_UPDATE_FRAME, COMMIT, until=RESPONSE_DONE), _step(COMMIT, until="closed")),
         )
         assert session.transcripts == (blocked, blocked), session
-        assert session.errors == (MISSING_TURN_DETECTION_TYPE, GUARDRAIL_VIOLATION, GUARDRAIL_VIOLATION), session
+        assert session.errors == (GUARDRAIL_VIOLATION, GUARDRAIL_VIOLATION), session
         assert session.types[-1] == "closed", session
         assert session.close_code == GUARDRAIL_END_CLOSE, session
         assert _sent_types(observed) == (
@@ -1948,7 +2333,7 @@ def test_non_string_transcript_field_closes_a_voice_session_with_the_same_proxy_
         assert session.types == (
             "session.created",
             "session.updated",
-            "error",
+            "session.updated",
             TRANSCRIPT_COMPLETED,
             "error",
             "closed",